logstash-filter-grok 4.1.1 → 4.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +14 -0
  3. data/Gemfile +6 -0
  4. data/LICENSE +199 -10
  5. data/README.md +1 -1
  6. data/docs/index.asciidoc +66 -0
  7. data/lib/logstash/filters/grok.rb +158 -46
  8. data/logstash-filter-grok.gemspec +4 -4
  9. data/spec/filters/grok_performance_spec.rb +144 -0
  10. data/spec/filters/grok_spec.rb +629 -646
  11. data/spec/spec_helper.rb +19 -0
  12. metadata +33 -10
data/spec/filters/grok_spec.rb CHANGED
@@ -1,392 +1,369 @@
1
1
  # encoding: utf-8
2
- require "logstash/devutils/rspec/spec_helper"
3
- require "stud/temporary"
2
+ require_relative "../spec_helper"
4
3
 
5
- module LogStash::Environment
6
- # running the grok code outside a logstash package means
7
- # LOGSTASH_HOME will not be defined, so let's set it here
8
- # before requiring the grok filter
9
- unless self.const_defined?(:LOGSTASH_HOME)
10
- LOGSTASH_HOME = File.expand_path("../../../", __FILE__)
4
+ require "logstash/filters/grok"
5
+
6
+ describe LogStash::Filters::Grok do
7
+ subject { described_class.new(config) }
8
+ let(:config) { {} }
9
+ let(:event) { LogStash::Event.new(data) }
10
+ let(:data) { { "message" => message } }
11
+
12
+ before(:each) do
13
+ subject.register
14
+ subject.filter(event)
11
15
  end
12
16
 
13
- # also :pattern_path method must exist so we define it too
14
- unless self.method_defined?(:pattern_path)
15
- def pattern_path(path)
16
- ::File.join(LOGSTASH_HOME, "patterns", path)
17
+ def self.sample(message, &block)
18
+ # mod = RSpec::Core::MemoizedHelpers.module_for(self)
19
+ # mod.attr_reader :message
20
+ # # mod.__send__(:define_method, :message) { message }
21
+ # it("matches: #{message}") { @message = message; block.call }
22
+ describe message do
23
+ let(:message) { message }
24
+ it("groks", &block)
17
25
  end
18
26
  end
19
- end
20
27
 
21
- require "logstash/filters/grok"
28
+ describe "simple syslog line" do
29
+ let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "overwrite" => [ "message" ] } }
30
+ let(:message) { 'Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]' }
31
+
32
+ it "matches pattern" do
33
+ expect( event.get("tags") ).to be nil
34
+ expect( event.get("logsource") ).to eql "evita"
35
+ expect( event.get("timestamp") ).to eql "Mar 16 00:01:25"
36
+ expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
37
+ expect( event.get("program") ).to eql "postfix/smtpd"
38
+ expect( event.get("pid") ).to eql "1713"
39
+ end
40
+
41
+ %w(v1 v8).each do |ecs_mode|
42
+ context "in ecs mode #{ecs_mode}" do
43
+ let(:config) { super().merge('ecs_compatibility' => ecs_mode) }
44
+
45
+ it "matches pattern" do
46
+ expect( event.get("host") ).to eql "hostname"=>"evita"
47
+ expect( event.get("process") ).to eql "name"=>"postfix/smtpd", "pid"=>1713
48
+ expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
49
+ end
50
+ end
51
+ end
22
52
 
23
- describe LogStash::Filters::Grok do
53
+ context 'with target' do
54
+ let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "grok" } }
24
55
 
25
- describe "simple syslog line" do
26
- # The logstash config goes here.
27
- # At this time, only filters are supported.
28
- config <<-CONFIG
29
- filter {
30
- grok {
31
- match => { "message" => "%{SYSLOGLINE}" }
32
- overwrite => [ "message" ]
33
- }
34
- }
35
- CONFIG
56
+ it "matches pattern" do
57
+ expect( event.get("message") ).to eql message
58
+ expect( event.get("tags") ).to be nil
59
+ expect( event.get("grok") ).to_not be nil
60
+ expect( event.get("[grok][timestamp]") ).to eql "Mar 16 00:01:25"
61
+ expect( event.get("[grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
62
+ expect( event.get("[grok][pid]") ).to eql "1713"
63
+ end
64
+ end
36
65
 
37
- sample "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]" do
38
- insist { subject.get("tags") }.nil?
39
- insist { subject.get("logsource") } == "evita"
40
- insist { subject.get("timestamp") } == "Mar 16 00:01:25"
41
- insist { subject.get("message") } == "connect from camomile.cloud9.net[168.100.1.3]"
42
- insist { subject.get("program") } == "postfix/smtpd"
43
- insist { subject.get("pid") } == "1713"
66
+ context 'with [deep] target' do
67
+ let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "[@metadata][grok]" } }
68
+
69
+ it "matches pattern" do
70
+ expect( event.get("message") ).to eql message
71
+ expect( event.get("tags") ).to be nil
72
+ expect( event.get("grok") ).to be nil
73
+ expect( event.get("[@metadata][grok][logsource]") ).to eql "evita"
74
+ expect( event.get("[@metadata][grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
75
+ end
44
76
  end
45
77
  end
46
78
 
47
79
  describe "ietf 5424 syslog line" do
48
- # The logstash config goes here.
49
- # At this time, only filters are supported.
50
- config <<-CONFIG
51
- filter {
52
- grok {
53
- match => { "message" => "%{SYSLOG5424LINE}" }
54
- }
55
- }
56
- CONFIG
80
+ let(:config) { { "match" => { "message" => "%{SYSLOG5424LINE}" } } }
57
81
 
58
82
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\"bar\"][id2 baz=\"something\"] Hello, syslog." do
59
- insist { subject.get("tags") }.nil?
60
- insist { subject.get("syslog5424_pri") } == "191"
61
- insist { subject.get("syslog5424_ver") } == "1"
62
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
63
- insist { subject.get("syslog5424_host") } == "paxton.local"
64
- insist { subject.get("syslog5424_app") } == "grokdebug"
65
- insist { subject.get("syslog5424_proc") } == "4123"
66
- insist { subject.get("syslog5424_msgid") } == nil
67
- insist { subject.get("syslog5424_sd") } == "[id1 foo=\"bar\"][id2 baz=\"something\"]"
68
- insist { subject.get("syslog5424_msg") } == "Hello, syslog."
83
+ expect( event.get("tags") ).to be nil
84
+ expect( event.get("syslog5424_pri") ).to eql "191"
85
+ expect( event.get("syslog5424_ver") ).to eql "1"
86
+ expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
87
+ expect( event.get("syslog5424_host") ).to eql "paxton.local"
88
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
89
+ expect( event.get("syslog5424_proc") ).to eql "4123"
90
+ expect( event.get("syslog5424_msgid") ).to be nil
91
+ expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"][id2 baz=\"something\"]"
92
+ expect( event.get("syslog5424_msg") ).to eql "Hello, syslog."
69
93
  end
70
94
 
71
95
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - [id1 foo=\"bar\"] No process ID." do
72
- insist { subject.get("tags") }.nil?
73
- insist { subject.get("syslog5424_pri") } == "191"
74
- insist { subject.get("syslog5424_ver") } == "1"
75
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
76
- insist { subject.get("syslog5424_host") } == "paxton.local"
77
- insist { subject.get("syslog5424_app") } == "grokdebug"
78
- insist { subject.get("syslog5424_proc") } == nil
79
- insist { subject.get("syslog5424_msgid") } == nil
80
- insist { subject.get("syslog5424_sd") } == "[id1 foo=\"bar\"]"
81
- insist { subject.get("syslog5424_msg") } == "No process ID."
96
+ expect( event.get("tags") ).to be nil
97
+ expect( event.get("syslog5424_pri") ).to eql "191"
98
+ expect( event.get("syslog5424_ver") ).to eql "1"
99
+ expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
100
+ expect( event.get("syslog5424_host") ).to eql "paxton.local"
101
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
102
+ expect( event.get("syslog5424_proc") ).to be nil
103
+ expect( event.get("syslog5424_msgid") ).to be nil
104
+ expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"]"
105
+ expect( event.get("syslog5424_msg") ).to eql "No process ID."
82
106
  end
83
107
 
84
108
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - - No structured data." do
85
- insist { subject.get("tags") }.nil?
86
- insist { subject.get("syslog5424_pri") } == "191"
87
- insist { subject.get("syslog5424_ver") } == "1"
88
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
89
- insist { subject.get("syslog5424_host") } == "paxton.local"
90
- insist { subject.get("syslog5424_app") } == "grokdebug"
91
- insist { subject.get("syslog5424_proc") } == "4123"
92
- insist { subject.get("syslog5424_msgid") } == nil
93
- insist { subject.get("syslog5424_sd") } == nil
94
- insist { subject.get("syslog5424_msg") } == "No structured data."
109
+ expect( event.get("tags") ).to be nil
110
+ expect( event.get("syslog5424_pri") ).to eql "191"
111
+ expect( event.get("syslog5424_ver") ).to eql "1"
112
+ expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
113
+ expect( event.get("syslog5424_host") ).to eql "paxton.local"
114
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
115
+ expect( event.get("syslog5424_proc") ).to eql '4123'
116
+ expect( event.get("syslog5424_msgid") ).to be nil
117
+ expect( event.get("syslog5424_sd") ).to be nil
118
+ expect( event.get("syslog5424_msg") ).to eql "No structured data."
95
119
  end
96
120
 
97
121
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - - No PID or SD." do
98
- insist { subject.get("tags") }.nil?
99
- insist { subject.get("syslog5424_pri") } == "191"
100
- insist { subject.get("syslog5424_ver") } == "1"
101
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
102
- insist { subject.get("syslog5424_host") } == "paxton.local"
103
- insist { subject.get("syslog5424_app") } == "grokdebug"
104
- insist { subject.get("syslog5424_proc") } == nil
105
- insist { subject.get("syslog5424_msgid") } == nil
106
- insist { subject.get("syslog5424_sd") } == nil
107
- insist { subject.get("syslog5424_msg") } == "No PID or SD."
122
+ expect( event.get("tags") ).to be nil
123
+ expect( event.get("syslog5424_pri") ).to eql "191"
124
+ expect( event.get("syslog5424_ver") ).to eql "1"
125
+ expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
126
+ expect( event.get("syslog5424_host") ).to eql "paxton.local"
127
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
128
+ expect( event.get("syslog5424_proc") ).to be nil
129
+ expect( event.get("syslog5424_msgid") ).to be nil
130
+ expect( event.get("syslog5424_sd") ).to be nil
131
+ expect( event.get("syslog5424_msg") ).to eql "No PID or SD."
108
132
  end
109
133
 
110
134
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - Missing structured data." do
111
- insist { subject.get("tags") }.nil?
112
- insist { subject.get("syslog5424_pri") } == "191"
113
- insist { subject.get("syslog5424_ver") } == "1"
114
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
115
- insist { subject.get("syslog5424_host") } == "paxton.local"
116
- insist { subject.get("syslog5424_app") } == "grokdebug"
117
- insist { subject.get("syslog5424_proc") } == "4123"
118
- insist { subject.get("syslog5424_msgid") } == nil
119
- insist { subject.get("syslog5424_sd") } == nil
120
- insist { subject.get("syslog5424_msg") } == "Missing structured data."
135
+ expect( event.get("tags") ).to be nil
136
+
137
+ expect( event.get("syslog5424_proc") ).to eql '4123'
138
+ expect( event.get("syslog5424_msgid") ).to be nil
139
+ expect( event.get("syslog5424_sd") ).to be nil
140
+ expect( event.get("syslog5424_msg") ).to eql "Missing structured data."
121
141
  end
122
142
 
123
143
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - - Additional spaces." do
124
- insist { subject.get("tags") }.nil?
125
- insist { subject.get("syslog5424_pri") } == "191"
126
- insist { subject.get("syslog5424_ver") } == "1"
127
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
128
- insist { subject.get("syslog5424_host") } == "paxton.local"
129
- insist { subject.get("syslog5424_app") } == "grokdebug"
130
- insist { subject.get("syslog5424_proc") } == "4123"
131
- insist { subject.get("syslog5424_msgid") } == nil
132
- insist { subject.get("syslog5424_sd") } == nil
133
- insist { subject.get("syslog5424_msg") } == "Additional spaces."
144
+ expect( event.get("tags") ).to be nil
145
+
146
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
147
+ expect( event.get("syslog5424_proc") ).to eql '4123'
148
+ expect( event.get("syslog5424_msgid") ).to be nil
149
+ expect( event.get("syslog5424_sd") ).to be nil
150
+ expect( event.get("syslog5424_msg") ).to eql "Additional spaces."
134
151
  end
135
152
 
136
153
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - Additional spaces and missing SD." do
137
- insist { subject.get("tags") }.nil?
138
- insist { subject.get("syslog5424_pri") } == "191"
139
- insist { subject.get("syslog5424_ver") } == "1"
140
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
141
- insist { subject.get("syslog5424_host") } == "paxton.local"
142
- insist { subject.get("syslog5424_app") } == "grokdebug"
143
- insist { subject.get("syslog5424_proc") } == "4123"
144
- insist { subject.get("syslog5424_msgid") } == nil
145
- insist { subject.get("syslog5424_sd") } == nil
146
- insist { subject.get("syslog5424_msg") } == "Additional spaces and missing SD."
154
+ expect( event.get("tags") ).to be nil
155
+
156
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
157
+ expect( event.get("syslog5424_proc") ).to eql '4123'
158
+ expect( event.get("syslog5424_msgid") ).to be nil
159
+ expect( event.get("syslog5424_sd") ).to be nil
160
+ expect( event.get("syslog5424_msg") ).to eql "Additional spaces and missing SD."
147
161
  end
148
162
 
149
163
  sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 dnsmasq-dhcp 8048 - - Appname contains a dash" do
150
- insist { subject.get("tags") }.nil?
151
- insist { subject.get("syslog5424_pri") } == "30"
152
- insist { subject.get("syslog5424_ver") } == "1"
153
- insist { subject.get("syslog5424_ts") } == "2014-04-04T16:44:07+02:00"
154
- insist { subject.get("syslog5424_host") } == "osctrl01"
155
- insist { subject.get("syslog5424_app") } == "dnsmasq-dhcp"
156
- insist { subject.get("syslog5424_proc") } == "8048"
157
- insist { subject.get("syslog5424_msgid") } == nil
158
- insist { subject.get("syslog5424_sd") } == nil
159
- insist { subject.get("syslog5424_msg") } == "Appname contains a dash"
164
+ expect( event.get("tags") ).to be nil
165
+ expect( event.get("syslog5424_pri") ).to eql "30"
166
+ expect( event.get("syslog5424_ver") ).to eql "1"
167
+ expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
168
+ expect( event.get("syslog5424_host") ).to eql "osctrl01"
169
+ expect( event.get("syslog5424_app") ).to eql "dnsmasq-dhcp"
170
+ expect( event.get("syslog5424_proc") ).to eql "8048"
171
+ expect( event.get("syslog5424_msgid") ).to be nil
172
+ expect( event.get("syslog5424_sd") ).to be nil
173
+ expect( event.get("syslog5424_msg") ).to eql "Appname contains a dash"
160
174
  end
161
175
 
162
176
  sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 - 8048 - - Appname is nil" do
163
- insist { subject.get("tags") }.nil?
164
- insist { subject.get("syslog5424_pri") } == "30"
165
- insist { subject.get("syslog5424_ver") } == "1"
166
- insist { subject.get("syslog5424_ts") } == "2014-04-04T16:44:07+02:00"
167
- insist { subject.get("syslog5424_host") } == "osctrl01"
168
- insist { subject.get("syslog5424_app") } == nil
169
- insist { subject.get("syslog5424_proc") } == "8048"
170
- insist { subject.get("syslog5424_msgid") } == nil
171
- insist { subject.get("syslog5424_sd") } == nil
172
- insist { subject.get("syslog5424_msg") } == "Appname is nil"
177
+ expect( event.get("tags") ).to be nil
178
+ expect( event.get("syslog5424_pri") ).to eql "30"
179
+ expect( event.get("syslog5424_ver") ).to eql "1"
180
+ expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
181
+ expect( event.get("syslog5424_host") ).to eql "osctrl01"
182
+ expect( event.get("syslog5424_app") ).to be nil
183
+ expect( event.get("syslog5424_proc") ).to eql "8048"
184
+ expect( event.get("syslog5424_msgid") ).to be nil
185
+ expect( event.get("syslog5424_sd") ).to be nil
186
+ expect( event.get("syslog5424_msg") ).to eql "Appname is nil"
173
187
  end
174
188
  end
175
189
 
176
- describe "parsing an event with multiple messages (array of strings)", :if => false do
177
- config <<-CONFIG
178
- filter {
179
- grok {
180
- match => { "message" => "(?:hello|world) %{NUMBER}" }
181
- named_captures_only => false
182
- }
183
- }
184
- CONFIG
190
+ describe "parsing an event with multiple messages (array of strings)", if: false do
191
+ let(:config) { { "message" => "(?:hello|world) %{NUMBER}" } }
192
+ let(:message) { [ "hello 12345", "world 23456" ] }
185
193
 
186
- sample("message" => [ "hello 12345", "world 23456" ]) do
187
- insist { subject.get("NUMBER") } == [ "12345", "23456" ]
194
+ it "matches them all" do
195
+ expect( event.get("NUMBER") ).to eql [ "12345", "23456" ]
188
196
  end
189
197
  end
190
198
 
191
199
  describe "coercing matched values" do
192
- config <<-CONFIG
193
- filter {
194
- grok {
195
- match => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" }
196
- }
197
- }
198
- CONFIG
200
+ let(:config) { { "match" => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" } } }
201
+ let(:message) { '400 454.33' }
199
202
 
200
- sample "400 454.33" do
201
- insist { subject.get("foo") } == 400
202
- insist { subject.get("foo") }.is_a?(Fixnum)
203
- insist { subject.get("bar") } == 454.33
204
- insist { subject.get("bar") }.is_a?(Float)
203
+ it "coerces matched values" do
204
+ expect( event.get("foo") ).to be_a Integer
205
+ expect( event.get("foo") ).to eql 400
206
+ expect( event.get("bar") ).to be_a Float
207
+ expect( event.get("bar") ).to eql 454.33
205
208
  end
206
209
  end
207
210
 
208
211
  describe "in-line pattern definitions" do
209
- config <<-CONFIG
210
- filter {
211
- grok {
212
- match => { "message" => "%{FIZZLE=\\d+}" }
213
- named_captures_only => false
214
- }
215
- }
216
- CONFIG
212
+ let(:config) { { "match" => { "message" => "%{FIZZLE=\\d+}" }, "named_captures_only" => false } }
217
213
 
218
214
  sample "hello 1234" do
219
- insist { subject.get("FIZZLE") } == "1234"
215
+ expect( event.get("FIZZLE") ).to eql '1234'
220
216
  end
221
217
  end
222
218
 
223
219
  describe "processing selected fields" do
224
- config <<-CONFIG
225
- filter {
226
- grok {
227
- match => { "message" => "%{WORD:word}" }
228
- match => { "examplefield" => "%{NUMBER:num}" }
229
- break_on_match => false
230
- }
220
+ let(:config) {
221
+ {
222
+ 'match' => { "message" => "%{WORD:word}", "examplefield" => "%{NUMBER:num}" },
223
+ 'break_on_match' => false
231
224
  }
232
- CONFIG
225
+ }
226
+ let(:data) { { "message" => "hello world", "examplefield" => "12345" } }
233
227
 
234
- sample("message" => "hello world", "examplefield" => "12345") do
235
- insist { subject.get("examplefield") } == "12345"
236
- insist { subject.get("word") } == "hello"
228
+ it "processes declared matches" do
229
+ expect( event.get("word") ).to eql 'hello'
230
+ expect( event.get("examplefield") ).to eql '12345'
237
231
  end
238
232
  end
239
233
 
240
234
  describe "adding fields on match" do
241
- config <<-CONFIG
242
- filter {
243
- grok {
244
- match => { "message" => "matchme %{NUMBER:fancy}" }
245
- add_field => [ "new_field", "%{fancy}" ]
246
- }
235
+ let(:config) {
236
+ {
237
+ 'match' => { "message" => "matchme %{NUMBER:fancy}" },
238
+ 'add_field' => [ "new_field", "%{fancy}" ]
247
239
  }
248
- CONFIG
240
+ }
249
241
 
250
242
  sample "matchme 1234" do
251
- insist { subject.get("tags") }.nil?
252
- insist { subject.get("new_field") } == "1234"
243
+ expect( event.get("tags") ).to be nil
244
+ expect( event.get("new_field") ).to eql "1234"
253
245
  end
254
246
 
255
247
  sample "this will not be matched" do
256
- insist { subject.get("tags") }.include?("_grokparsefailure")
257
- reject { subject }.include?("new_field")
248
+ expect( event.get("tags") ).to include("_grokparsefailure")
249
+ expect( event ).not_to include 'new_field'
258
250
  end
259
251
  end
260
252
 
261
253
  context "empty fields" do
262
254
  describe "drop by default" do
263
- config <<-CONFIG
264
- filter {
265
- grok {
266
- match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
267
- }
255
+ let(:config) {
256
+ {
257
+ 'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
268
258
  }
269
- CONFIG
259
+ }
270
260
 
271
261
  sample "1=test" do
272
- insist { subject.get("tags") }.nil?
273
- insist { subject }.include?("foo1")
262
+ expect( event.get("tags") ).to be nil
263
+ expect( event ).to include 'foo1'
274
264
 
275
265
  # Since 'foo2' was not captured, it must not be present in the event.
276
- reject { subject }.include?("foo2")
266
+ expect( event ).not_to include 'foo2'
277
267
  end
278
268
  end
279
269
 
280
270
  describe "keep if keep_empty_captures is true" do
281
- config <<-CONFIG
282
- filter {
283
- grok {
284
- match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
285
- keep_empty_captures => true
286
- }
271
+ let(:config) {
272
+ {
273
+ 'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" },
274
+ 'keep_empty_captures' => true
287
275
  }
288
- CONFIG
276
+ }
289
277
 
290
278
  sample "1=test" do
291
- insist { subject.get("tags") }.nil?
279
+ expect( event.get("tags") ).to be nil
292
280
  # use .to_hash for this test, for now, because right now
293
281
  # the Event.include? returns false for missing fields as well
294
282
  # as for fields with nil values.
295
- insist { subject.to_hash }.include?("foo2")
296
- insist { subject.to_hash }.include?("foo2")
283
+ expect( event.to_hash ).to include 'foo1'
284
+ expect( event.to_hash ).to include 'foo2'
297
285
  end
298
286
  end
299
287
  end
300
288
 
301
289
  describe "when named_captures_only == false" do
302
- config <<-CONFIG
303
- filter {
304
- grok {
305
- match => { "message" => "Hello %{WORD}. %{WORD:foo}" }
306
- named_captures_only => false
307
- }
290
+ let(:config) {
291
+ {
292
+ 'match' => { "message" => "Hello %{WORD}. %{WORD:foo}" },
293
+ 'named_captures_only' => false
308
294
  }
309
- CONFIG
295
+ }
310
296
 
311
297
  sample "Hello World, yo!" do
312
- insist { subject }.include?("WORD")
313
- insist { subject.get("WORD") } == "World"
314
- insist { subject }.include?("foo")
315
- insist { subject.get("foo") } == "yo"
298
+ expect( event ).to include 'WORD'
299
+ expect( event.get("WORD") ).to eql "World"
300
+ expect( event ).to include 'foo'
301
+ expect( event.get("foo") ).to eql "yo"
316
302
  end
317
303
  end
318
304
 
319
305
  describe "using oniguruma named captures (?<name>regex)" do
320
306
  context "plain regexp" do
321
- config <<-'CONFIG'
322
- filter {
323
- grok {
324
- match => { "message" => "(?<foo>\w+)" }
325
- }
307
+ let(:config) {
308
+ {
309
+ 'match' => { "message" => "(?<foo>\\w+)" }
326
310
  }
327
- CONFIG
311
+ }
312
+
328
313
  sample "hello world" do
329
- insist { subject.get("tags") }.nil?
330
- insist { subject.get("foo") } == "hello"
314
+ expect( event.get("tags") ).to be nil
315
+ expect( event.get("foo") ).to eql "hello"
331
316
  end
332
317
  end
333
318
 
334
319
  context "grok patterns" do
335
- config <<-'CONFIG'
336
- filter {
337
- grok {
338
- match => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
339
- }
320
+ let(:config) {
321
+ {
322
+ 'match' => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
340
323
  }
341
- CONFIG
324
+ }
342
325
 
343
326
  sample "fancy 12-12-12 12:12:12" do
344
- insist { subject.get("tags") }.nil?
345
- insist { subject.get("timestamp") } == "12-12-12 12:12:12"
327
+ expect( event.get("tags") ).to be nil
328
+ expect( event.get("timestamp") ).to eql "12-12-12 12:12:12"
346
329
  end
347
330
  end
348
331
  end
349
332
 
350
333
  describe "grok on integer types" do
351
- config <<-'CONFIG'
352
- filter {
353
- grok {
354
- match => { "status" => "^403$" }
355
- add_tag => "four_oh_three"
356
- }
334
+ let(:config) {
335
+ {
336
+ 'match' => { "status" => "^403$" }, 'add_tag' => "four_oh_three"
357
337
  }
358
- CONFIG
338
+ }
339
+ let(:data) { Hash({ "status" => 403 }) }
359
340
 
360
- sample("status" => 403) do
361
- reject { subject.get("tags") }.include?("_grokparsefailure")
362
- insist { subject.get("tags") }.include?("four_oh_three")
341
+ it "parses" do
342
+ expect( event.get("tags") ).not_to include "_grokparsefailure"
343
+ expect( event.get("tags") ).to include "four_oh_three"
363
344
  end
364
345
  end
365
346
 
366
347
  describe "grok on float types" do
367
- config <<-'CONFIG'
368
- filter {
369
- grok {
370
- match => { "version" => "^1.0$" }
371
- add_tag => "one_point_oh"
372
- }
348
+ let(:config) {
349
+ {
350
+ 'match' => { "version" => "^1.0$" }, 'add_tag' => "one_point_oh"
373
351
  }
374
- CONFIG
352
+ }
353
+ let(:data) { Hash({ "version" => 1.0 }) }
375
354
 
376
- sample("version" => 1.0) do
377
- insist { subject.get("tags") }.include?("one_point_oh")
378
- insist { subject.get("tags") }.include?("one_point_oh")
355
+ it "parses" do
356
+ expect( event.get("tags") ).not_to include "_grokparsefailure"
357
+ expect( event.get("tags") ).to include "one_point_oh"
379
358
  end
380
359
  end
381
360
 
382
361
  describe "grok on %{LOGLEVEL}" do
383
- config <<-'CONFIG'
384
- filter {
385
- grok {
386
- match => { "message" => "%{LOGLEVEL:level}: error!" }
387
- }
362
+ let(:config) {
363
+ {
364
+ 'match' => { "message" => "%{LOGLEVEL:level}: error!" }
388
365
  }
389
- CONFIG
366
+ }
390
367
 
391
368
  log_level_names = %w(
392
369
  trace Trace TRACE
@@ -402,547 +379,553 @@ describe LogStash::Filters::Grok do
402
379
  )
403
380
  log_level_names.each do |level_name|
404
381
  sample "#{level_name}: error!" do
405
- insist { subject.get("level") } == level_name
382
+ expect( event.get("level") ).to eql level_name
406
383
  end
407
384
  end
408
385
  end
409
386
 
410
387
  describe "timeout on failure" do
411
- config <<-CONFIG
412
- filter {
413
- grok {
414
- match => {
415
- message => "(.*a){30}"
416
- }
417
- timeout_millis => 100
418
- }
388
+ let(:config) {
389
+ {
390
+ 'match' => { "message" => "(.*a){30}" },
391
+ 'timeout_millis' => 100
419
392
  }
420
- CONFIG
393
+ }
421
394
 
422
395
  sample "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" do
423
- expect(subject.get("tags")).to include("_groktimeout")
424
- expect(subject.get("tags")).not_to include("_grokparsefailure")
396
+ expect( event.get("tags") ).to include("_groktimeout")
397
+ expect( event.get("tags") ).not_to include("_grokparsefailure")
398
+ end
399
+ end
400
+
401
+ describe "no timeout on failure with multiple patterns (when timeout not grouped)" do
402
+ let(:config) {
403
+ {
404
+ 'match' => {
405
+ "message" => [
406
+ "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
407
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
408
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
409
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
410
+ "(.*a){20}"
411
+ ]
412
+ },
413
+ 'timeout_millis' => 750,
414
+ 'timeout_scope' => 'pattern'
415
+ }
416
+ }
417
+
418
+ sample( 'b' * 15 + 'c' * 15 + 'd' * 15 + 'e' * 15 + ' ' + 'a' * 20 ) do
419
+ expect( event.get("tags") ).to be nil
420
+ end
421
+ end
422
+
423
+ describe "timeout on grouped (multi-pattern) failure" do
424
+ let(:config) {
425
+ {
426
+ 'match' => {
427
+ "message" => [
428
+ "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
429
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
430
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
431
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
432
+ "(.*a){20}"
433
+ ]
434
+ },
435
+ 'timeout_millis' => 750,
436
+ 'timeout_scope' => 'event'
437
+ }
438
+ }
439
+
440
+ sample( 'b' * 15 + 'c' * 15 + 'd' * 15 + 'e' * 15 + ' ' + 'a' * 20 ) do
441
+ expect( event.get("tags") ).to include("_groktimeout")
442
+ expect( event.get("tags") ).not_to include("_grokparsefailure")
425
443
  end
426
444
  end
427
445
 
428
446
  describe "tagging on failure" do
429
- config <<-CONFIG
430
- filter {
431
- grok {
432
- match => { "message" => "matchme %{NUMBER:fancy}" }
433
- tag_on_failure => not_a_match
434
- }
447
+ let(:config) {
448
+ {
449
+ 'match' => { "message" => "matchme %{NUMBER:fancy}" },
450
+ 'tag_on_failure' => 'not_a_match'
435
451
  }
436
- CONFIG
452
+ }
437
453
 
438
454
  sample "matchme 1234" do
439
- insist { subject.get("tags") }.nil?
455
+ expect( event.get("tags") ).to be nil
440
456
  end
441
457
 
442
458
  sample "this will not be matched" do
443
- insist { subject.get("tags") }.include?("not_a_match")
459
+ expect( event.get("tags") ).to include("not_a_match")
444
460
  end
445
461
  end
446
462
 
447
463
  describe "captures named fields even if the whole text matches" do
448
- config <<-CONFIG
449
- filter {
450
- grok {
451
- match => { "message" => "%{DATE_EU:stimestamp}" }
452
- }
464
+ let(:config) {
465
+ {
466
+ 'match' => { "message" => "%{DATE_EU:stimestamp}" }
453
467
  }
454
- CONFIG
468
+ }
455
469
 
456
470
  sample "11/01/01" do
457
- insist { subject.get("stimestamp") } == "11/01/01"
471
+ expect( event.get("stimestamp") ).to eql "11/01/01"
458
472
  end
459
473
  end
460
474
 
461
475
  describe "allow dashes in capture names" do
462
- config <<-CONFIG
463
- filter {
464
- grok {
465
- match => { "message" => "%{WORD:foo-bar}" }
466
- }
476
+ let(:config) {
477
+ {
478
+ 'match' => { "message" => "%{WORD:foo-bar}" }
467
479
  }
468
- CONFIG
480
+ }
469
481
 
470
482
  sample "hello world" do
471
- insist { subject.get("foo-bar") } == "hello"
472
- end
473
- end
474
-
475
- describe "performance test", :performance => true do
476
- event_count = 100000
477
- min_rate = 2000
478
-
479
- max_duration = event_count / min_rate
480
- input = "Nov 24 01:29:01 -0800"
481
- config <<-CONFIG
482
- input {
483
- generator {
484
- count => #{event_count}
485
- message => "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
486
- }
487
- }
488
- filter {
489
- grok {
490
- match => { "message" => "%{SYSLOGLINE}" }
491
- overwrite => [ "message" ]
492
- }
493
- }
494
- output { null { } }
495
- CONFIG
496
-
497
- 2.times do
498
- start = Time.now
499
- agent do
500
- duration = (Time.now - start)
501
- puts "filters/grok parse rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s"
502
- insist { duration } < max_duration
503
- end
483
+ expect( event.get("foo-bar") ).to eql "hello"
504
484
  end
505
485
  end
506
486
 
507
487
  describe "single value match with duplicate-named fields in pattern" do
508
- config <<-CONFIG
509
- filter {
510
- grok {
511
- match => { "message" => "%{INT:foo}|%{WORD:foo}" }
512
- }
488
+ let(:config) {
489
+ {
490
+ 'match' => { "message" => "%{INT:foo}|%{WORD:foo}" }
513
491
  }
514
- CONFIG
492
+ }
515
493
 
516
494
  sample "hello world" do
517
- insist { subject.get("foo") }.is_a?(String)
495
+ expect( event.get("foo") ).to be_a(String)
518
496
  end
519
497
 
520
498
  sample "123 world" do
521
- insist { subject.get("foo") }.is_a?(String)
499
+ expect( event.get("foo") ).to be_a(String)
522
500
  end
523
501
  end
524
502
 
525
- describe "break_on_match default should be true and first match should exit filter" do
526
- config <<-CONFIG
527
- filter {
528
- grok {
529
- match => { "message" => "%{INT:foo}"
530
- "somefield" => "%{INT:bar}"}
531
- }
532
- }
533
- CONFIG
534
-
535
- sample("message" => "hello world 123", "somefield" => "testme abc 999") do
536
- insist { subject.get("foo") } == "123"
537
- insist { subject.get("bar") }.nil?
538
- end
539
- end
540
503
 
541
- describe "break_on_match when set to false should try all patterns" do
542
- config <<-CONFIG
543
- filter {
544
- grok {
545
- match => { "message" => "%{INT:foo}"
546
- "somefield" => "%{INT:bar}"}
547
- break_on_match => false
548
- }
504
+ describe "break_on_match default should be true" do
505
+ let(:config) {
506
+ {
507
+ 'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" }
549
508
  }
550
- CONFIG
509
+ }
510
+ let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
551
511
 
552
- sample("message" => "hello world 123", "somefield" => "testme abc 999") do
553
- insist { subject.get("foo") } == "123"
554
- insist { subject.get("bar") } == "999"
512
+ it 'exits filter after first match' do
513
+ expect( event.get("foo") ).to eql '123'
514
+ expect( event.get("bar") ).to be nil
555
515
  end
556
516
  end
557
517
 
558
- describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
559
- config <<-CONFIG
560
- filter {
561
- grok {
562
- match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
563
- break_on_match => false
564
- }
518
+ describe "break_on_match when set to false" do
519
+ let(:config) {
520
+ {
521
+ 'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" },
522
+ 'break_on_match' => false
565
523
  }
566
- CONFIG
567
-
568
- sample "treebranch" do
569
- insist { subject.get("name2") } == "branch"
570
- end
571
-
572
- sample "bushbeard" do
573
- insist { subject.get("name1") } == "bush"
574
- end
524
+ }
525
+ let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
575
526
 
576
- sample "treebeard" do
577
- insist { subject.get("name1") } == "tree"
578
- insist { subject.get("name2") } == "beard"
527
+ it 'should try all patterns' do
528
+ expect( event.get("foo") ).to eql '123'
529
+ expect( event.get("bar") ).to eql '999'
579
530
  end
580
531
  end
581
532
 
582
- describe "break_on_match default for array input with single grok pattern" do
583
- config <<-CONFIG
584
- filter {
585
- grok {
586
- match => { "message" => "%{INT:foo}"}
587
- }
533
+ context "break_on_match default for array input with single grok pattern" do
534
+ let(:config) {
535
+ {
536
+ 'match' => { "message" => "%{INT:foo}" },
537
+ 'break_on_match' => false
588
538
  }
589
- CONFIG
539
+ }
590
540
 
591
- # array input --
592
- sample("message" => ["hello world 123", "line 23"]) do
593
- insist { subject.get("foo") } == ["123", "23"]
594
- insist { subject.get("tags") }.nil?
541
+ describe 'fully matching input' do
542
+ let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
543
+ it 'matches' do
544
+ expect( event.get("foo") ).to eql ["123", "23"]
545
+ expect( event.get("tags") ).to be nil
546
+ end
595
547
  end
596
548
 
597
- # array input, one of them matches
598
- sample("message" => ["hello world 123", "abc"]) do
599
- insist { subject.get("foo") } == "123"
600
- insist { subject.get("tags") }.nil?
549
+ describe 'partially matching input' do
550
+ let(:data) { Hash("message" => ["hello world 123", "abc"]) } # array input, one of them matches
551
+ it 'matches' do
552
+ expect( event.get("foo") ).to eql "123"
553
+ expect( event.get("tags") ).to be nil
554
+ end
601
555
  end
602
556
  end
603
557
 
604
558
  describe "break_on_match = true (default) for array input with multiple grok pattern" do
605
- config <<-CONFIG
606
- filter {
607
- grok {
608
- match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
609
- }
559
+ let(:config) {
560
+ {
561
+ 'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
610
562
  }
611
- CONFIG
612
-
613
- # array input --
614
- sample("message" => ["hello world 123", "line 23"]) do
615
- insist { subject.get("foo") } == ["123", "23"]
616
- insist { subject.get("bar") }.nil?
617
- insist { subject.get("tags") }.nil?
563
+ }
564
+
565
+ describe 'matching input' do
566
+ let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
567
+ it 'matches' do
568
+ expect( event.get("foo") ).to eql ["123", "23"]
569
+ expect( event.get("bar") ).to be nil
570
+ expect( event.get("tags") ).to be nil
571
+ end
618
572
  end
619
573
 
620
- # array input, one of them matches
621
- sample("message" => ["hello world", "line 23"]) do
622
- insist { subject.get("bar") } == "hello"
623
- insist { subject.get("foo") } == "23"
624
- insist { subject.get("tags") }.nil?
574
+ describe 'partially matching input' do
575
+ let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
576
+ it 'matches' do
577
+ expect( event.get("bar") ).to eql 'hello'
578
+ expect( event.get("foo") ).to eql "23"
579
+ expect( event.get("tags") ).to be nil
580
+ end
625
581
  end
626
582
  end
627
583
 
628
584
  describe "break_on_match = false for array input with multiple grok pattern" do
629
- config <<-CONFIG
630
- filter {
631
- grok {
632
- match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
633
- break_on_match => false
634
- }
585
+ let(:config) {
586
+ {
587
+ 'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] },
588
+ 'break_on_match' => false
635
589
  }
636
- CONFIG
637
-
638
- # array input --
639
- sample("message" => ["hello world 123", "line 23"]) do
640
- insist { subject.get("foo") } == ["123", "23"]
641
- insist { subject.get("bar") } == ["hello", "line"]
642
- insist { subject.get("tags") }.nil?
590
+ }
591
+
592
+ describe 'fully matching input' do
593
+ let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
594
+ it 'matches' do
595
+ expect( event.get("foo") ).to eql ["123", "23"]
596
+ expect( event.get("bar") ).to eql ["hello", "line"]
597
+ expect( event.get("tags") ).to be nil
598
+ end
643
599
  end
644
600
 
645
- # array input, one of them matches
646
- sample("message" => ["hello world", "line 23"]) do
647
- insist { subject.get("bar") } == ["hello", "line"]
648
- insist { subject.get("foo") } == "23"
649
- insist { subject.get("tags") }.nil?
601
+ describe 'partially matching input' do
602
+ let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
603
+ it 'matches' do
604
+ expect( event.get("bar") ).to eql ["hello", "line"]
605
+ expect( event.get("foo") ).to eql "23"
606
+ expect( event.get("tags") ).to be nil
607
+ end
650
608
  end
651
609
  end
652
610
 
653
611
  describe "grok with unicode" do
654
- config <<-CONFIG
655
- filter {
656
- grok {
657
- #match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
658
- match => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
659
- }
612
+ let(:config) {
613
+ {
614
+ #'match' => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
615
+ 'match' => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
660
616
  }
661
- CONFIG
617
+ }
662
618
 
663
619
  sample "<22>Jan 4 07:50:46 mailmaster postfix/policy-spf[9454]: : SPF permerror (Junk encountered in record 'v=spf1 mx a:mail.domain.no ip4:192.168.0.4 �all'): Envelope-from: email@domain.no" do
664
- insist { subject.get("tags") }.nil?
665
- insist { subject.get("syslog_pri") } == "22"
666
- insist { subject.get("syslog_program") } == "postfix/policy-spf"
620
+ expect( event.get("tags") ).to be nil
621
+ expect( event.get("syslog_pri") ).to eql "22"
622
+ expect( event.get("syslog_program") ).to eql "postfix/policy-spf"
667
623
  end
668
624
  end
669
625
 
670
- describe "patterns in the 'patterns/' dir override core patterns" do
671
-
672
- let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
673
- let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
626
+ describe "grok with nil coerced value" do
627
+ let(:config) {
628
+ {
629
+ 'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
630
+ }
631
+ }
674
632
 
675
- before do
676
- FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
677
- @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
678
- @file.write('WORD \b[2-5]\b')
679
- @file.close
633
+ sample "test 28.4ms" do
634
+ expect( event.get("duration") ).to eql 28.4
635
+ expect( event.get("tags") ).to be nil
680
636
  end
681
637
 
682
- let(:config) do
683
- 'filter { grok { match => { "message" => "%{WORD:word}" } } }'
638
+ sample "test N/A" do
639
+ expect( event.to_hash ).not_to include("duration")
640
+ expect( event.get("tags") ).to be nil
684
641
  end
685
642
 
686
- sample("message" => 'hello') do
687
- insist { subject.get("tags") } == ["_grokparsefailure"]
643
+ sample "test abc" do
644
+ expect( event.get("duration") ).to be nil
645
+ expect( event.get("tags") ).to eql ["_grokparsefailure"]
688
646
  end
647
+ end
689
648
 
690
- after do
691
- File.unlink @file
692
- FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
649
+ describe "grok with nil coerced value and keep_empty_captures" do
650
+ let(:config) {
651
+ {
652
+ 'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" },
653
+ 'keep_empty_captures' => true
654
+ }
655
+ }
656
+
657
+ sample "test N/A" do
658
+ expect( event.to_hash ).to include("duration")
659
+ expect( event.get("tags") ).to be nil
693
660
  end
694
661
  end
695
662
 
696
- describe "patterns in custom dir override those in 'patterns/' dir" do
697
-
698
- let(:tmpdir) { Stud::Temporary.directory }
699
- let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
700
- let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
663
+ describe "grok with no coercion" do
664
+ let(:config) {
665
+ {
666
+ 'match' => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" },
667
+ }
668
+ }
701
669
 
702
- before do
703
- FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
704
- @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
705
- @file1.write('WORD \b[2-5]\b')
706
- @file1.close
707
- @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
708
- @file2.write('WORD \b[0-1]\b')
709
- @file2.close
670
+ sample "test 28.4ms" do
671
+ expect( event.get("duration") ).to eql '28.4'
672
+ expect( event.get("tags") ).to be nil
710
673
  end
711
674
 
712
- let(:config) do
713
- "filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
675
+ sample "test N/A" do
676
+ expect( event.get("duration") ).to be nil
677
+ expect( event.get("tags") ).to be nil
714
678
  end
679
+ end
715
680
 
716
- sample("message" => '0') do
717
- insist { subject.get("tags") } == nil
718
- end
681
+ describe "opening/closing" do
682
+ let(:config) { { "match" => {"message" => "A"} } }
683
+ let(:message) { 'AAA' }
719
684
 
720
- after do
721
- File.unlink @file1
722
- File.unlink @file2
723
- FileUtils.remove_entry tmpdir
724
- FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
685
+ it "should close cleanly" do
686
+ expect { subject.do_close }.not_to raise_error
725
687
  end
726
688
  end
727
689
 
728
- describe "patterns with file glob" do
729
-
730
- let(:tmpdir) { Stud::Temporary.directory }
690
+ describe "after grok when the event is JSON serialised the field values are unchanged" do
691
+ let(:config) {
692
+ {
693
+ 'match' => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"],
694
+ 'remove_field' => ["message","severity"],
695
+ 'add_tag' => ["ssh_failure"]
696
+ }
697
+ }
731
698
 
732
- before do
733
- @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
734
- @file3.write('WORD \b[0-1]\b')
735
- @file3.close
736
- @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
737
- @file4.write('WORD \b[2-5]\b')
738
- @file4.close
739
- end
699
+ sample('{"facility":"auth","message":"Failed password for testuser from 1.1.1.1 port 22"}') do
700
+ expect( event.get("username") ).to eql "testuser"
701
+ expect( event.get("port") ).to eql "22"
702
+ expect( event.get("src_ip") ).to eql "1.1.1.1"
703
+ expect( LogStash::Json.dump(event.get('username')) ).to eql "\"testuser\""
740
704
 
741
- let(:config) do
742
- "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
705
+ expect( event.to_json ).to match %r|"src_ip":"1.1.1.1"|
706
+ expect( event.to_json ).to match %r|"@timestamp":"#{Regexp.escape(event.get('@timestamp').to_s)}"|
707
+ expect( event.to_json ).to match %r|"port":"22"|
708
+ expect( event.to_json ).to match %r|"@version":"1"|
709
+ expect( event.to_json ).to match %r|"username"|i
710
+ expect( event.to_json ).to match %r|"testuser"|
711
+ expect( event.to_json ).to match %r|"tags":\["ssh_failure"\]|
743
712
  end
713
+ end
744
714
 
745
- sample("message" => '0') do
746
- insist { subject.get("tags") } == nil
747
- end
715
+ describe "grok with inline pattern definition successfully extracts fields" do
716
+ let(:config) {
717
+ {
718
+ 'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" },
719
+ 'pattern_definitions' => {
720
+ "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
721
+ "MY_PATTERN" => "%{YEAR}"
722
+ }
723
+ }
724
+ }
748
725
 
749
- after do
750
- File.unlink @file3
751
- File.unlink @file4
752
- FileUtils.remove_entry tmpdir
726
+ sample "Mon Dec 26 16:22:08 2016 error 2020" do
727
+ expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
728
+ expect( event.get("level") ).to eql "error"
729
+ expect( event.get("hindsight") ).to eql "2020"
753
730
  end
754
731
  end
755
732
 
756
- describe "patterns with file glob on directory that contains subdirectories" do
757
-
758
- let(:tmpdir) { Stud::Temporary.directory }
733
+ describe "grok with inline pattern definition overwrites existing pattern definition" do
734
+ let(:config) {
735
+ {
736
+ 'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" },
737
+ # loglevel was previously ([Aa]lert|ALERT|[Tt]...
738
+ 'pattern_definitions' => {
739
+ "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
740
+ "LOGLEVEL" => "%{NUMBER}"
741
+ }
742
+ }
743
+ }
759
744
 
760
- before do
761
- @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
762
- @file3.write('WORD \b[0-1]\b')
763
- @file3.close
764
- Dir.mkdir(File.join(tmpdir, "subdir"))
745
+ sample "Mon Dec 26 16:22:08 2016 9999" do
746
+ expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
747
+ expect( event.get("level") ).to eql "9999"
765
748
  end
749
+ end
766
750
 
751
+ context 'when timeouts are explicitly disabled' do
767
752
  let(:config) do
768
- "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
753
+ {
754
+ "timeout_millis" => 0
755
+ }
769
756
  end
770
757
 
771
- sample("message" => '0') do
772
- insist { subject.get("tags") } == nil
773
- end
758
+ context 'when given a pathological input', slow: true do
759
+ let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
760
+ let(:config) { super().merge("match" => { "message" => "(.*a){30}" }) }
774
761
 
775
- after do
776
- File.unlink @file3
777
- FileUtils.remove_entry tmpdir
762
+ it 'blocks for at least 3 seconds' do
763
+ blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
764
+ expect do
765
+ Timeout.timeout(3, blocking_exception_class) do
766
+ subject.filter(event)
767
+ end
768
+ end.to raise_exception(blocking_exception_class)
769
+ end
778
770
  end
779
771
  end
772
+ end
780
773
 
781
- describe "grok with nil coerced value" do
782
- config <<-CONFIG
783
- filter {
784
- grok {
785
- match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
786
- }
787
- }
788
- CONFIG
774
+ describe LogStash::Filters::Grok do
789
775
 
790
- sample "test 28.4ms" do
791
- insist { subject.get("duration") } == 28.4
792
- insist { subject.get("tags") }.nil?
793
- end
776
+ subject(:grok_filter) { described_class.new(config) }
777
+ let(:config) { {} }
794
778
 
795
- sample "test N/A" do
796
- insist { insist { subject.to_hash }.include?("duration") }.fails
797
- insist { subject.get("tags") }.nil?
798
- end
779
+ context 'when initialized with `ecs_compatibility => v8`' do
780
+ let(:config) { super().merge("ecs_compatibility" => "v8", "match" => ["message", "%{SYSLOGLINE}"]) }
781
+ context '#register' do
782
+ let(:logger_stub) { double('Logger').as_null_object }
783
+ before(:each) { allow_any_instance_of(described_class).to receive(:logger).and_return(logger_stub)}
799
784
 
800
- sample "test abc" do
801
- insist { subject.get("duration") }.nil?
802
- insist { subject.get("tags") } == ["_grokparsefailure"]
785
+ it 'logs a helpful warning about the unreleased v8' do
786
+ grok_filter.register
787
+
788
+ expect(logger_stub).to have_received(:warn).with(a_string_including "preview of the unreleased ECS v8")
789
+ end
803
790
  end
804
791
  end
792
+ end
805
793
 
806
- describe "grok with nil coerced value and keep_empty_captures" do
807
- config <<-CONFIG
808
- filter {
809
- grok {
810
- match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
811
- keep_empty_captures => true
812
- }
813
- }
814
- CONFIG
794
+ describe LogStash::Filters::Grok do
795
+ describe "(LEGACY)" do
796
+ describe "patterns in the 'patterns/' dir override core patterns" do
815
797
 
816
- sample "test N/A" do
817
- insist { subject.to_hash }.include?("duration")
818
- insist { subject.get("tags") }.nil?
819
- end
798
+ let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
799
+ let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
820
800
 
821
- end
801
+ before do
802
+ FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
803
+ @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
804
+ @file.write('WORD \b[2-5]\b')
805
+ @file.close
806
+ end
822
807
 
823
- describe "grok with no coercion" do
824
- config <<-CONFIG
825
- filter {
826
- grok {
827
- match => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" }
828
- }
829
- }
830
- CONFIG
808
+ let(:config) do
809
+ 'filter { grok { match => { "message" => "%{WORD:word}" } } }'
810
+ end
831
811
 
832
- sample "test 28.4ms" do
833
- insist { subject.get("duration") } == "28.4"
834
- insist { subject.get("tags") }.nil?
835
- end
812
+ sample("message" => 'hello') do
813
+ expect(subject.get("tags")).to eql ["_grokparsefailure"]
814
+ end
836
815
 
837
- sample "test N/A" do
838
- insist { subject.get("duration") }.nil?
839
- insist { subject.get("tags") }.nil?
816
+ after do
817
+ File.unlink @file
818
+ FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
819
+ end
840
820
  end
841
- end
842
821
 
843
- describe "opening/closing" do
844
- let(:config) { {"match" => {"message" => "A"}} }
845
- subject(:plugin) do
846
- ::LogStash::Filters::Grok.new(config)
847
- end
822
+ describe "patterns in custom dir override those in 'patterns/' dir" do
848
823
 
849
- before do
850
- plugin.register
851
- end
824
+ let(:tmpdir) { Stud::Temporary.directory }
825
+ let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
826
+ let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
852
827
 
853
- it "should close cleanly" do
854
- expect { plugin.do_close }.not_to raise_error
855
- end
856
- end
828
+ before do
829
+ FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
830
+ @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
831
+ @file1.write('WORD \b[2-5]\b')
832
+ @file1.close
833
+ @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
834
+ @file2.write('WORD \b[0-1]\b')
835
+ @file2.close
836
+ end
857
837
 
858
- describe "after grok when the event is JSON serialised the field values are unchanged" do
859
- config <<-CONFIG
860
- filter {grok {match => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"] remove_field => ["message","severity"] add_tag => ["ssh_failure"]}}
861
- CONFIG
838
+ let(:config) do
839
+ "filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
840
+ end
862
841
 
863
- sample('{"facility":"auth","message":"Failed password for testuser from 1.1.1.1 port 22"}') do
864
- insist { subject.get("username") } == "testuser"
865
- insist { subject.get("port") } == "22"
866
- insist { subject.get("src_ip") } == "1.1.1.1"
867
- insist { LogStash::Json.dump(subject.get('username')) } == "\"testuser\""
842
+ sample("message" => '0') do
843
+ expect(subject.get("tags")).to be nil
844
+ end
868
845
 
869
- insist { subject.to_json } =~ %r|"src_ip":"1.1.1.1"|
870
- insist { subject.to_json } =~ %r|"@timestamp":"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ"|
871
- insist { subject.to_json } =~ %r|"port":"22"|
872
- insist { subject.to_json } =~ %r|"@version":"1"|
873
- insist { subject.to_json } =~ %r|"username"|i
874
- insist { subject.to_json } =~ %r|"testuser"|
875
- insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]|
846
+ after do
847
+ File.unlink @file1
848
+ File.unlink @file2
849
+ FileUtils.remove_entry tmpdir
850
+ FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
851
+ end
876
852
  end
877
- end
878
853
 
879
- describe "grok with inline pattern definition successfully extracts fields" do
880
- config <<-CONFIG
881
- filter {
882
- grok {
883
- match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" }
884
- pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
885
- "MY_PATTERN" => "%{YEAR}"}
886
- }
887
- }
888
- CONFIG
854
+ describe "patterns with file glob" do
889
855
 
890
- sample "Mon Dec 26 16:22:08 2016 error 2020" do
891
- insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
892
- insist { subject.get("level") } == "error"
893
- insist { subject.get("hindsight") } == "2020"
894
- end
895
- end
856
+ let(:tmpdir) { Stud::Temporary.directory }
896
857
 
897
- describe "grok with inline pattern definition overwrites existing pattern definition" do
898
- config <<-CONFIG
899
- filter {
900
- grok {
901
- match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" }
902
- # loglevel was previously ([Aa]lert|ALERT|[Tt]...
903
- pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
904
- "LOGLEVEL" => "%{NUMBER}"}
905
- }
906
- }
907
- CONFIG
858
+ before do
859
+ @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
860
+ @file3.write('WORD \b[0-1]\b')
861
+ @file3.close
862
+ @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
863
+ @file4.write('WORD \b[2-5]\b')
864
+ @file4.close
865
+ end
908
866
 
909
- sample "Mon Dec 26 16:22:08 2016 9999" do
910
- insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
911
- insist { subject.get("level") } == "9999"
912
- end
913
- end
867
+ let(:config) do
868
+ "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
869
+ end
914
870
 
871
+ sample("message" => '0') do
872
+ expect(subject.get("tags")).to be nil
873
+ end
915
874
 
916
- describe "direct plugin testing" do
917
- subject do
918
- plugin = LogStash::Filters::Grok.new(options)
919
- plugin.register
920
- plugin
875
+ after do
876
+ File.unlink @file3
877
+ File.unlink @file4
878
+ FileUtils.remove_entry tmpdir
879
+ end
921
880
  end
922
881
 
923
- let(:data) { {"message" => message} }
924
- let(:event) { LogStash::Event.new(data) }
882
+ describe "patterns with file glob on directory that contains subdirectories" do
925
883
 
926
- context 'when timeouts are explicitly disabled' do
927
- let(:options) do
928
- {
929
- "timeout_millis" => 0
884
+ let(:tmpdir) { Stud::Temporary.directory }
885
+
886
+ before do
887
+ @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
888
+ @file3.write('WORD \b[0-1]\b')
889
+ @file3.close
890
+ Dir.mkdir(File.join(tmpdir, "subdir"))
891
+ end
892
+
893
+ let(:config) do
894
+ "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
895
+ end
896
+
897
+ sample("message" => '0') do
898
+ expect(subject.get("tags")).to be nil
899
+ end
900
+
901
+ after do
902
+ File.unlink @file3
903
+ FileUtils.remove_entry tmpdir
904
+ end
905
+ end
906
+
907
+ describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
908
+ config <<-CONFIG
909
+ filter {
910
+ grok {
911
+ match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
912
+ break_on_match => false
930
913
  }
914
+ }
915
+ CONFIG
916
+
917
+ sample "treebranch" do
918
+ expect(subject.get("name2")).to eql "branch"
931
919
  end
932
920
 
933
- context 'when given a pathological input' do
934
- let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
935
- let(:options) { super().merge("match" => { "message" => "(.*a){30}" }) }
921
+ sample "bushbeard" do
922
+ expect(subject.get("name1")).to eql "bush"
923
+ end
936
924
 
937
- it 'blocks for at least 3 seconds' do
938
- blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
939
- expect do
940
- Timeout.timeout(3, blocking_exception_class) do
941
- subject.filter(event)
942
- end
943
- end.to raise_exception(blocking_exception_class)
944
- end
925
+ sample "treebeard" do
926
+ expect(subject.get("name1")).to eql "tree"
927
+ expect(subject.get("name2")).to eql "beard"
945
928
  end
946
929
  end
947
930
  end
948
- end
931
+ end