logstash-filter-grok 4.1.1 → 4.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +14 -0
  3. data/Gemfile +6 -0
  4. data/LICENSE +199 -10
  5. data/README.md +1 -1
  6. data/docs/index.asciidoc +66 -0
  7. data/lib/logstash/filters/grok.rb +158 -46
  8. data/logstash-filter-grok.gemspec +4 -4
  9. data/spec/filters/grok_performance_spec.rb +144 -0
  10. data/spec/filters/grok_spec.rb +629 -646
  11. data/spec/spec_helper.rb +19 -0
  12. metadata +33 -10
data/spec/filters/grok_spec.rb CHANGED
@@ -1,392 +1,369 @@
1
1
  # encoding: utf-8
2
- require "logstash/devutils/rspec/spec_helper"
3
- require "stud/temporary"
2
+ require_relative "../spec_helper"
4
3
 
5
- module LogStash::Environment
6
- # running the grok code outside a logstash package means
7
- # LOGSTASH_HOME will not be defined, so let's set it here
8
- # before requiring the grok filter
9
- unless self.const_defined?(:LOGSTASH_HOME)
10
- LOGSTASH_HOME = File.expand_path("../../../", __FILE__)
4
+ require "logstash/filters/grok"
5
+
6
+ describe LogStash::Filters::Grok do
7
+ subject { described_class.new(config) }
8
+ let(:config) { {} }
9
+ let(:event) { LogStash::Event.new(data) }
10
+ let(:data) { { "message" => message } }
11
+
12
+ before(:each) do
13
+ subject.register
14
+ subject.filter(event)
11
15
  end
12
16
 
13
- # also :pattern_path method must exist so we define it too
14
- unless self.method_defined?(:pattern_path)
15
- def pattern_path(path)
16
- ::File.join(LOGSTASH_HOME, "patterns", path)
17
+ def self.sample(message, &block)
18
+ # mod = RSpec::Core::MemoizedHelpers.module_for(self)
19
+ # mod.attr_reader :message
20
+ # # mod.__send__(:define_method, :message) { message }
21
+ # it("matches: #{message}") { @message = message; block.call }
22
+ describe message do
23
+ let(:message) { message }
24
+ it("groks", &block)
17
25
  end
18
26
  end
19
- end
20
27
 
21
- require "logstash/filters/grok"
28
+ describe "simple syslog line" do
29
+ let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "overwrite" => [ "message" ] } }
30
+ let(:message) { 'Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]' }
31
+
32
+ it "matches pattern" do
33
+ expect( event.get("tags") ).to be nil
34
+ expect( event.get("logsource") ).to eql "evita"
35
+ expect( event.get("timestamp") ).to eql "Mar 16 00:01:25"
36
+ expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
37
+ expect( event.get("program") ).to eql "postfix/smtpd"
38
+ expect( event.get("pid") ).to eql "1713"
39
+ end
40
+
41
+ %w(v1 v8).each do |ecs_mode|
42
+ context "in ecs mode #{ecs_mode}" do
43
+ let(:config) { super().merge('ecs_compatibility' => ecs_mode) }
44
+
45
+ it "matches pattern" do
46
+ expect( event.get("host") ).to eql "hostname"=>"evita"
47
+ expect( event.get("process") ).to eql "name"=>"postfix/smtpd", "pid"=>1713
48
+ expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
49
+ end
50
+ end
51
+ end
22
52
 
23
- describe LogStash::Filters::Grok do
53
+ context 'with target' do
54
+ let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "grok" } }
24
55
 
25
- describe "simple syslog line" do
26
- # The logstash config goes here.
27
- # At this time, only filters are supported.
28
- config <<-CONFIG
29
- filter {
30
- grok {
31
- match => { "message" => "%{SYSLOGLINE}" }
32
- overwrite => [ "message" ]
33
- }
34
- }
35
- CONFIG
56
+ it "matches pattern" do
57
+ expect( event.get("message") ).to eql message
58
+ expect( event.get("tags") ).to be nil
59
+ expect( event.get("grok") ).to_not be nil
60
+ expect( event.get("[grok][timestamp]") ).to eql "Mar 16 00:01:25"
61
+ expect( event.get("[grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
62
+ expect( event.get("[grok][pid]") ).to eql "1713"
63
+ end
64
+ end
36
65
 
37
- sample "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]" do
38
- insist { subject.get("tags") }.nil?
39
- insist { subject.get("logsource") } == "evita"
40
- insist { subject.get("timestamp") } == "Mar 16 00:01:25"
41
- insist { subject.get("message") } == "connect from camomile.cloud9.net[168.100.1.3]"
42
- insist { subject.get("program") } == "postfix/smtpd"
43
- insist { subject.get("pid") } == "1713"
66
+ context 'with [deep] target' do
67
+ let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "[@metadata][grok]" } }
68
+
69
+ it "matches pattern" do
70
+ expect( event.get("message") ).to eql message
71
+ expect( event.get("tags") ).to be nil
72
+ expect( event.get("grok") ).to be nil
73
+ expect( event.get("[@metadata][grok][logsource]") ).to eql "evita"
74
+ expect( event.get("[@metadata][grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
75
+ end
44
76
  end
45
77
  end
46
78
 
47
79
  describe "ietf 5424 syslog line" do
48
- # The logstash config goes here.
49
- # At this time, only filters are supported.
50
- config <<-CONFIG
51
- filter {
52
- grok {
53
- match => { "message" => "%{SYSLOG5424LINE}" }
54
- }
55
- }
56
- CONFIG
80
+ let(:config) { { "match" => { "message" => "%{SYSLOG5424LINE}" } } }
57
81
 
58
82
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\"bar\"][id2 baz=\"something\"] Hello, syslog." do
59
- insist { subject.get("tags") }.nil?
60
- insist { subject.get("syslog5424_pri") } == "191"
61
- insist { subject.get("syslog5424_ver") } == "1"
62
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
63
- insist { subject.get("syslog5424_host") } == "paxton.local"
64
- insist { subject.get("syslog5424_app") } == "grokdebug"
65
- insist { subject.get("syslog5424_proc") } == "4123"
66
- insist { subject.get("syslog5424_msgid") } == nil
67
- insist { subject.get("syslog5424_sd") } == "[id1 foo=\"bar\"][id2 baz=\"something\"]"
68
- insist { subject.get("syslog5424_msg") } == "Hello, syslog."
83
+ expect( event.get("tags") ).to be nil
84
+ expect( event.get("syslog5424_pri") ).to eql "191"
85
+ expect( event.get("syslog5424_ver") ).to eql "1"
86
+ expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
87
+ expect( event.get("syslog5424_host") ).to eql "paxton.local"
88
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
89
+ expect( event.get("syslog5424_proc") ).to eql "4123"
90
+ expect( event.get("syslog5424_msgid") ).to be nil
91
+ expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"][id2 baz=\"something\"]"
92
+ expect( event.get("syslog5424_msg") ).to eql "Hello, syslog."
69
93
  end
70
94
 
71
95
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - [id1 foo=\"bar\"] No process ID." do
72
- insist { subject.get("tags") }.nil?
73
- insist { subject.get("syslog5424_pri") } == "191"
74
- insist { subject.get("syslog5424_ver") } == "1"
75
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
76
- insist { subject.get("syslog5424_host") } == "paxton.local"
77
- insist { subject.get("syslog5424_app") } == "grokdebug"
78
- insist { subject.get("syslog5424_proc") } == nil
79
- insist { subject.get("syslog5424_msgid") } == nil
80
- insist { subject.get("syslog5424_sd") } == "[id1 foo=\"bar\"]"
81
- insist { subject.get("syslog5424_msg") } == "No process ID."
96
+ expect( event.get("tags") ).to be nil
97
+ expect( event.get("syslog5424_pri") ).to eql "191"
98
+ expect( event.get("syslog5424_ver") ).to eql "1"
99
+ expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
100
+ expect( event.get("syslog5424_host") ).to eql "paxton.local"
101
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
102
+ expect( event.get("syslog5424_proc") ).to be nil
103
+ expect( event.get("syslog5424_msgid") ).to be nil
104
+ expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"]"
105
+ expect( event.get("syslog5424_msg") ).to eql "No process ID."
82
106
  end
83
107
 
84
108
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - - No structured data." do
85
- insist { subject.get("tags") }.nil?
86
- insist { subject.get("syslog5424_pri") } == "191"
87
- insist { subject.get("syslog5424_ver") } == "1"
88
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
89
- insist { subject.get("syslog5424_host") } == "paxton.local"
90
- insist { subject.get("syslog5424_app") } == "grokdebug"
91
- insist { subject.get("syslog5424_proc") } == "4123"
92
- insist { subject.get("syslog5424_msgid") } == nil
93
- insist { subject.get("syslog5424_sd") } == nil
94
- insist { subject.get("syslog5424_msg") } == "No structured data."
109
+ expect( event.get("tags") ).to be nil
110
+ expect( event.get("syslog5424_pri") ).to eql "191"
111
+ expect( event.get("syslog5424_ver") ).to eql "1"
112
+ expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
113
+ expect( event.get("syslog5424_host") ).to eql "paxton.local"
114
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
115
+ expect( event.get("syslog5424_proc") ).to eql '4123'
116
+ expect( event.get("syslog5424_msgid") ).to be nil
117
+ expect( event.get("syslog5424_sd") ).to be nil
118
+ expect( event.get("syslog5424_msg") ).to eql "No structured data."
95
119
  end
96
120
 
97
121
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - - No PID or SD." do
98
- insist { subject.get("tags") }.nil?
99
- insist { subject.get("syslog5424_pri") } == "191"
100
- insist { subject.get("syslog5424_ver") } == "1"
101
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
102
- insist { subject.get("syslog5424_host") } == "paxton.local"
103
- insist { subject.get("syslog5424_app") } == "grokdebug"
104
- insist { subject.get("syslog5424_proc") } == nil
105
- insist { subject.get("syslog5424_msgid") } == nil
106
- insist { subject.get("syslog5424_sd") } == nil
107
- insist { subject.get("syslog5424_msg") } == "No PID or SD."
122
+ expect( event.get("tags") ).to be nil
123
+ expect( event.get("syslog5424_pri") ).to eql "191"
124
+ expect( event.get("syslog5424_ver") ).to eql "1"
125
+ expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
126
+ expect( event.get("syslog5424_host") ).to eql "paxton.local"
127
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
128
+ expect( event.get("syslog5424_proc") ).to be nil
129
+ expect( event.get("syslog5424_msgid") ).to be nil
130
+ expect( event.get("syslog5424_sd") ).to be nil
131
+ expect( event.get("syslog5424_msg") ).to eql "No PID or SD."
108
132
  end
109
133
 
110
134
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - Missing structured data." do
111
- insist { subject.get("tags") }.nil?
112
- insist { subject.get("syslog5424_pri") } == "191"
113
- insist { subject.get("syslog5424_ver") } == "1"
114
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
115
- insist { subject.get("syslog5424_host") } == "paxton.local"
116
- insist { subject.get("syslog5424_app") } == "grokdebug"
117
- insist { subject.get("syslog5424_proc") } == "4123"
118
- insist { subject.get("syslog5424_msgid") } == nil
119
- insist { subject.get("syslog5424_sd") } == nil
120
- insist { subject.get("syslog5424_msg") } == "Missing structured data."
135
+ expect( event.get("tags") ).to be nil
136
+
137
+ expect( event.get("syslog5424_proc") ).to eql '4123'
138
+ expect( event.get("syslog5424_msgid") ).to be nil
139
+ expect( event.get("syslog5424_sd") ).to be nil
140
+ expect( event.get("syslog5424_msg") ).to eql "Missing structured data."
121
141
  end
122
142
 
123
143
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - - Additional spaces." do
124
- insist { subject.get("tags") }.nil?
125
- insist { subject.get("syslog5424_pri") } == "191"
126
- insist { subject.get("syslog5424_ver") } == "1"
127
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
128
- insist { subject.get("syslog5424_host") } == "paxton.local"
129
- insist { subject.get("syslog5424_app") } == "grokdebug"
130
- insist { subject.get("syslog5424_proc") } == "4123"
131
- insist { subject.get("syslog5424_msgid") } == nil
132
- insist { subject.get("syslog5424_sd") } == nil
133
- insist { subject.get("syslog5424_msg") } == "Additional spaces."
144
+ expect( event.get("tags") ).to be nil
145
+
146
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
147
+ expect( event.get("syslog5424_proc") ).to eql '4123'
148
+ expect( event.get("syslog5424_msgid") ).to be nil
149
+ expect( event.get("syslog5424_sd") ).to be nil
150
+ expect( event.get("syslog5424_msg") ).to eql "Additional spaces."
134
151
  end
135
152
 
136
153
  sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - Additional spaces and missing SD." do
137
- insist { subject.get("tags") }.nil?
138
- insist { subject.get("syslog5424_pri") } == "191"
139
- insist { subject.get("syslog5424_ver") } == "1"
140
- insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
141
- insist { subject.get("syslog5424_host") } == "paxton.local"
142
- insist { subject.get("syslog5424_app") } == "grokdebug"
143
- insist { subject.get("syslog5424_proc") } == "4123"
144
- insist { subject.get("syslog5424_msgid") } == nil
145
- insist { subject.get("syslog5424_sd") } == nil
146
- insist { subject.get("syslog5424_msg") } == "Additional spaces and missing SD."
154
+ expect( event.get("tags") ).to be nil
155
+
156
+ expect( event.get("syslog5424_app") ).to eql "grokdebug"
157
+ expect( event.get("syslog5424_proc") ).to eql '4123'
158
+ expect( event.get("syslog5424_msgid") ).to be nil
159
+ expect( event.get("syslog5424_sd") ).to be nil
160
+ expect( event.get("syslog5424_msg") ).to eql "Additional spaces and missing SD."
147
161
  end
148
162
 
149
163
  sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 dnsmasq-dhcp 8048 - - Appname contains a dash" do
150
- insist { subject.get("tags") }.nil?
151
- insist { subject.get("syslog5424_pri") } == "30"
152
- insist { subject.get("syslog5424_ver") } == "1"
153
- insist { subject.get("syslog5424_ts") } == "2014-04-04T16:44:07+02:00"
154
- insist { subject.get("syslog5424_host") } == "osctrl01"
155
- insist { subject.get("syslog5424_app") } == "dnsmasq-dhcp"
156
- insist { subject.get("syslog5424_proc") } == "8048"
157
- insist { subject.get("syslog5424_msgid") } == nil
158
- insist { subject.get("syslog5424_sd") } == nil
159
- insist { subject.get("syslog5424_msg") } == "Appname contains a dash"
164
+ expect( event.get("tags") ).to be nil
165
+ expect( event.get("syslog5424_pri") ).to eql "30"
166
+ expect( event.get("syslog5424_ver") ).to eql "1"
167
+ expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
168
+ expect( event.get("syslog5424_host") ).to eql "osctrl01"
169
+ expect( event.get("syslog5424_app") ).to eql "dnsmasq-dhcp"
170
+ expect( event.get("syslog5424_proc") ).to eql "8048"
171
+ expect( event.get("syslog5424_msgid") ).to be nil
172
+ expect( event.get("syslog5424_sd") ).to be nil
173
+ expect( event.get("syslog5424_msg") ).to eql "Appname contains a dash"
160
174
  end
161
175
 
162
176
  sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 - 8048 - - Appname is nil" do
163
- insist { subject.get("tags") }.nil?
164
- insist { subject.get("syslog5424_pri") } == "30"
165
- insist { subject.get("syslog5424_ver") } == "1"
166
- insist { subject.get("syslog5424_ts") } == "2014-04-04T16:44:07+02:00"
167
- insist { subject.get("syslog5424_host") } == "osctrl01"
168
- insist { subject.get("syslog5424_app") } == nil
169
- insist { subject.get("syslog5424_proc") } == "8048"
170
- insist { subject.get("syslog5424_msgid") } == nil
171
- insist { subject.get("syslog5424_sd") } == nil
172
- insist { subject.get("syslog5424_msg") } == "Appname is nil"
177
+ expect( event.get("tags") ).to be nil
178
+ expect( event.get("syslog5424_pri") ).to eql "30"
179
+ expect( event.get("syslog5424_ver") ).to eql "1"
180
+ expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
181
+ expect( event.get("syslog5424_host") ).to eql "osctrl01"
182
+ expect( event.get("syslog5424_app") ).to be nil
183
+ expect( event.get("syslog5424_proc") ).to eql "8048"
184
+ expect( event.get("syslog5424_msgid") ).to be nil
185
+ expect( event.get("syslog5424_sd") ).to be nil
186
+ expect( event.get("syslog5424_msg") ).to eql "Appname is nil"
173
187
  end
174
188
  end
175
189
 
176
- describe "parsing an event with multiple messages (array of strings)", :if => false do
177
- config <<-CONFIG
178
- filter {
179
- grok {
180
- match => { "message" => "(?:hello|world) %{NUMBER}" }
181
- named_captures_only => false
182
- }
183
- }
184
- CONFIG
190
+ describe "parsing an event with multiple messages (array of strings)", if: false do
191
+ let(:config) { { "message" => "(?:hello|world) %{NUMBER}" } }
192
+ let(:message) { [ "hello 12345", "world 23456" ] }
185
193
 
186
- sample("message" => [ "hello 12345", "world 23456" ]) do
187
- insist { subject.get("NUMBER") } == [ "12345", "23456" ]
194
+ it "matches them all" do
195
+ expect( event.get("NUMBER") ).to eql [ "12345", "23456" ]
188
196
  end
189
197
  end
190
198
 
191
199
  describe "coercing matched values" do
192
- config <<-CONFIG
193
- filter {
194
- grok {
195
- match => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" }
196
- }
197
- }
198
- CONFIG
200
+ let(:config) { { "match" => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" } } }
201
+ let(:message) { '400 454.33' }
199
202
 
200
- sample "400 454.33" do
201
- insist { subject.get("foo") } == 400
202
- insist { subject.get("foo") }.is_a?(Fixnum)
203
- insist { subject.get("bar") } == 454.33
204
- insist { subject.get("bar") }.is_a?(Float)
203
+ it "coerces matched values" do
204
+ expect( event.get("foo") ).to be_a Integer
205
+ expect( event.get("foo") ).to eql 400
206
+ expect( event.get("bar") ).to be_a Float
207
+ expect( event.get("bar") ).to eql 454.33
205
208
  end
206
209
  end
207
210
 
208
211
  describe "in-line pattern definitions" do
209
- config <<-CONFIG
210
- filter {
211
- grok {
212
- match => { "message" => "%{FIZZLE=\\d+}" }
213
- named_captures_only => false
214
- }
215
- }
216
- CONFIG
212
+ let(:config) { { "match" => { "message" => "%{FIZZLE=\\d+}" }, "named_captures_only" => false } }
217
213
 
218
214
  sample "hello 1234" do
219
- insist { subject.get("FIZZLE") } == "1234"
215
+ expect( event.get("FIZZLE") ).to eql '1234'
220
216
  end
221
217
  end
222
218
 
223
219
  describe "processing selected fields" do
224
- config <<-CONFIG
225
- filter {
226
- grok {
227
- match => { "message" => "%{WORD:word}" }
228
- match => { "examplefield" => "%{NUMBER:num}" }
229
- break_on_match => false
230
- }
220
+ let(:config) {
221
+ {
222
+ 'match' => { "message" => "%{WORD:word}", "examplefield" => "%{NUMBER:num}" },
223
+ 'break_on_match' => false
231
224
  }
232
- CONFIG
225
+ }
226
+ let(:data) { { "message" => "hello world", "examplefield" => "12345" } }
233
227
 
234
- sample("message" => "hello world", "examplefield" => "12345") do
235
- insist { subject.get("examplefield") } == "12345"
236
- insist { subject.get("word") } == "hello"
228
+ it "processes declared matches" do
229
+ expect( event.get("word") ).to eql 'hello'
230
+ expect( event.get("examplefield") ).to eql '12345'
237
231
  end
238
232
  end
239
233
 
240
234
  describe "adding fields on match" do
241
- config <<-CONFIG
242
- filter {
243
- grok {
244
- match => { "message" => "matchme %{NUMBER:fancy}" }
245
- add_field => [ "new_field", "%{fancy}" ]
246
- }
235
+ let(:config) {
236
+ {
237
+ 'match' => { "message" => "matchme %{NUMBER:fancy}" },
238
+ 'add_field' => [ "new_field", "%{fancy}" ]
247
239
  }
248
- CONFIG
240
+ }
249
241
 
250
242
  sample "matchme 1234" do
251
- insist { subject.get("tags") }.nil?
252
- insist { subject.get("new_field") } == "1234"
243
+ expect( event.get("tags") ).to be nil
244
+ expect( event.get("new_field") ).to eql "1234"
253
245
  end
254
246
 
255
247
  sample "this will not be matched" do
256
- insist { subject.get("tags") }.include?("_grokparsefailure")
257
- reject { subject }.include?("new_field")
248
+ expect( event.get("tags") ).to include("_grokparsefailure")
249
+ expect( event ).not_to include 'new_field'
258
250
  end
259
251
  end
260
252
 
261
253
  context "empty fields" do
262
254
  describe "drop by default" do
263
- config <<-CONFIG
264
- filter {
265
- grok {
266
- match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
267
- }
255
+ let(:config) {
256
+ {
257
+ 'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
268
258
  }
269
- CONFIG
259
+ }
270
260
 
271
261
  sample "1=test" do
272
- insist { subject.get("tags") }.nil?
273
- insist { subject }.include?("foo1")
262
+ expect( event.get("tags") ).to be nil
263
+ expect( event ).to include 'foo1'
274
264
 
275
265
  # Since 'foo2' was not captured, it must not be present in the event.
276
- reject { subject }.include?("foo2")
266
+ expect( event ).not_to include 'foo2'
277
267
  end
278
268
  end
279
269
 
280
270
  describe "keep if keep_empty_captures is true" do
281
- config <<-CONFIG
282
- filter {
283
- grok {
284
- match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
285
- keep_empty_captures => true
286
- }
271
+ let(:config) {
272
+ {
273
+ 'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" },
274
+ 'keep_empty_captures' => true
287
275
  }
288
- CONFIG
276
+ }
289
277
 
290
278
  sample "1=test" do
291
- insist { subject.get("tags") }.nil?
279
+ expect( event.get("tags") ).to be nil
292
280
  # use .to_hash for this test, for now, because right now
293
281
  # the Event.include? returns false for missing fields as well
294
282
  # as for fields with nil values.
295
- insist { subject.to_hash }.include?("foo2")
296
- insist { subject.to_hash }.include?("foo2")
283
+ expect( event.to_hash ).to include 'foo1'
284
+ expect( event.to_hash ).to include 'foo2'
297
285
  end
298
286
  end
299
287
  end
300
288
 
301
289
  describe "when named_captures_only == false" do
302
- config <<-CONFIG
303
- filter {
304
- grok {
305
- match => { "message" => "Hello %{WORD}. %{WORD:foo}" }
306
- named_captures_only => false
307
- }
290
+ let(:config) {
291
+ {
292
+ 'match' => { "message" => "Hello %{WORD}. %{WORD:foo}" },
293
+ 'named_captures_only' => false
308
294
  }
309
- CONFIG
295
+ }
310
296
 
311
297
  sample "Hello World, yo!" do
312
- insist { subject }.include?("WORD")
313
- insist { subject.get("WORD") } == "World"
314
- insist { subject }.include?("foo")
315
- insist { subject.get("foo") } == "yo"
298
+ expect( event ).to include 'WORD'
299
+ expect( event.get("WORD") ).to eql "World"
300
+ expect( event ).to include 'foo'
301
+ expect( event.get("foo") ).to eql "yo"
316
302
  end
317
303
  end
318
304
 
319
305
  describe "using oniguruma named captures (?<name>regex)" do
320
306
  context "plain regexp" do
321
- config <<-'CONFIG'
322
- filter {
323
- grok {
324
- match => { "message" => "(?<foo>\w+)" }
325
- }
307
+ let(:config) {
308
+ {
309
+ 'match' => { "message" => "(?<foo>\\w+)" }
326
310
  }
327
- CONFIG
311
+ }
312
+
328
313
  sample "hello world" do
329
- insist { subject.get("tags") }.nil?
330
- insist { subject.get("foo") } == "hello"
314
+ expect( event.get("tags") ).to be nil
315
+ expect( event.get("foo") ).to eql "hello"
331
316
  end
332
317
  end
333
318
 
334
319
  context "grok patterns" do
335
- config <<-'CONFIG'
336
- filter {
337
- grok {
338
- match => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
339
- }
320
+ let(:config) {
321
+ {
322
+ 'match' => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
340
323
  }
341
- CONFIG
324
+ }
342
325
 
343
326
  sample "fancy 12-12-12 12:12:12" do
344
- insist { subject.get("tags") }.nil?
345
- insist { subject.get("timestamp") } == "12-12-12 12:12:12"
327
+ expect( event.get("tags") ).to be nil
328
+ expect( event.get("timestamp") ).to eql "12-12-12 12:12:12"
346
329
  end
347
330
  end
348
331
  end
349
332
 
350
333
  describe "grok on integer types" do
351
- config <<-'CONFIG'
352
- filter {
353
- grok {
354
- match => { "status" => "^403$" }
355
- add_tag => "four_oh_three"
356
- }
334
+ let(:config) {
335
+ {
336
+ 'match' => { "status" => "^403$" }, 'add_tag' => "four_oh_three"
357
337
  }
358
- CONFIG
338
+ }
339
+ let(:data) { Hash({ "status" => 403 }) }
359
340
 
360
- sample("status" => 403) do
361
- reject { subject.get("tags") }.include?("_grokparsefailure")
362
- insist { subject.get("tags") }.include?("four_oh_three")
341
+ it "parses" do
342
+ expect( event.get("tags") ).not_to include "_grokparsefailure"
343
+ expect( event.get("tags") ).to include "four_oh_three"
363
344
  end
364
345
  end
365
346
 
366
347
  describe "grok on float types" do
367
- config <<-'CONFIG'
368
- filter {
369
- grok {
370
- match => { "version" => "^1.0$" }
371
- add_tag => "one_point_oh"
372
- }
348
+ let(:config) {
349
+ {
350
+ 'match' => { "version" => "^1.0$" }, 'add_tag' => "one_point_oh"
373
351
  }
374
- CONFIG
352
+ }
353
+ let(:data) { Hash({ "version" => 1.0 }) }
375
354
 
376
- sample("version" => 1.0) do
377
- insist { subject.get("tags") }.include?("one_point_oh")
378
- insist { subject.get("tags") }.include?("one_point_oh")
355
+ it "parses" do
356
+ expect( event.get("tags") ).not_to include "_grokparsefailure"
357
+ expect( event.get("tags") ).to include "one_point_oh"
379
358
  end
380
359
  end
381
360
 
382
361
  describe "grok on %{LOGLEVEL}" do
383
- config <<-'CONFIG'
384
- filter {
385
- grok {
386
- match => { "message" => "%{LOGLEVEL:level}: error!" }
387
- }
362
+ let(:config) {
363
+ {
364
+ 'match' => { "message" => "%{LOGLEVEL:level}: error!" }
388
365
  }
389
- CONFIG
366
+ }
390
367
 
391
368
  log_level_names = %w(
392
369
  trace Trace TRACE
@@ -402,547 +379,553 @@ describe LogStash::Filters::Grok do
402
379
  )
403
380
  log_level_names.each do |level_name|
404
381
  sample "#{level_name}: error!" do
405
- insist { subject.get("level") } == level_name
382
+ expect( event.get("level") ).to eql level_name
406
383
  end
407
384
  end
408
385
  end
409
386
 
410
387
  describe "timeout on failure" do
411
- config <<-CONFIG
412
- filter {
413
- grok {
414
- match => {
415
- message => "(.*a){30}"
416
- }
417
- timeout_millis => 100
418
- }
388
+ let(:config) {
389
+ {
390
+ 'match' => { "message" => "(.*a){30}" },
391
+ 'timeout_millis' => 100
419
392
  }
420
- CONFIG
393
+ }
421
394
 
422
395
  sample "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" do
423
- expect(subject.get("tags")).to include("_groktimeout")
424
- expect(subject.get("tags")).not_to include("_grokparsefailure")
396
+ expect( event.get("tags") ).to include("_groktimeout")
397
+ expect( event.get("tags") ).not_to include("_grokparsefailure")
398
+ end
399
+ end
400
+
401
+ describe "no timeout on failure with multiple patterns (when timeout not grouped)" do
402
+ let(:config) {
403
+ {
404
+ 'match' => {
405
+ "message" => [
406
+ "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
407
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
408
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
409
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
410
+ "(.*a){20}"
411
+ ]
412
+ },
413
+ 'timeout_millis' => 750,
414
+ 'timeout_scope' => 'pattern'
415
+ }
416
+ }
417
+
418
+ sample( 'b' * 15 + 'c' * 15 + 'd' * 15 + 'e' * 15 + ' ' + 'a' * 20 ) do
419
+ expect( event.get("tags") ).to be nil
420
+ end
421
+ end
422
+
423
+ describe "timeout on grouped (multi-pattern) failure" do
424
+ let(:config) {
425
+ {
426
+ 'match' => {
427
+ "message" => [
428
+ "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
429
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
430
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
431
+ "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
432
+ "(.*a){20}"
433
+ ]
434
+ },
435
+ 'timeout_millis' => 750,
436
+ 'timeout_scope' => 'event'
437
+ }
438
+ }
439
+
440
+ sample( 'b' * 15 + 'c' * 15 + 'd' * 15 + 'e' * 15 + ' ' + 'a' * 20 ) do
441
+ expect( event.get("tags") ).to include("_groktimeout")
442
+ expect( event.get("tags") ).not_to include("_grokparsefailure")
425
443
  end
426
444
  end
427
445
 
428
446
  describe "tagging on failure" do
429
- config <<-CONFIG
430
- filter {
431
- grok {
432
- match => { "message" => "matchme %{NUMBER:fancy}" }
433
- tag_on_failure => not_a_match
434
- }
447
+ let(:config) {
448
+ {
449
+ 'match' => { "message" => "matchme %{NUMBER:fancy}" },
450
+ 'tag_on_failure' => 'not_a_match'
435
451
  }
436
- CONFIG
452
+ }
437
453
 
438
454
  sample "matchme 1234" do
439
- insist { subject.get("tags") }.nil?
455
+ expect( event.get("tags") ).to be nil
440
456
  end
441
457
 
442
458
  sample "this will not be matched" do
443
- insist { subject.get("tags") }.include?("not_a_match")
459
+ expect( event.get("tags") ).to include("not_a_match")
444
460
  end
445
461
  end
446
462
 
447
463
  describe "captures named fields even if the whole text matches" do
448
- config <<-CONFIG
449
- filter {
450
- grok {
451
- match => { "message" => "%{DATE_EU:stimestamp}" }
452
- }
464
+ let(:config) {
465
+ {
466
+ 'match' => { "message" => "%{DATE_EU:stimestamp}" }
453
467
  }
454
- CONFIG
468
+ }
455
469
 
456
470
  sample "11/01/01" do
457
- insist { subject.get("stimestamp") } == "11/01/01"
471
+ expect( event.get("stimestamp") ).to eql "11/01/01"
458
472
  end
459
473
  end
460
474
 
461
475
  describe "allow dashes in capture names" do
462
- config <<-CONFIG
463
- filter {
464
- grok {
465
- match => { "message" => "%{WORD:foo-bar}" }
466
- }
476
+ let(:config) {
477
+ {
478
+ 'match' => { "message" => "%{WORD:foo-bar}" }
467
479
  }
468
- CONFIG
480
+ }
469
481
 
470
482
  sample "hello world" do
471
- insist { subject.get("foo-bar") } == "hello"
472
- end
473
- end
474
-
475
- describe "performance test", :performance => true do
476
- event_count = 100000
477
- min_rate = 2000
478
-
479
- max_duration = event_count / min_rate
480
- input = "Nov 24 01:29:01 -0800"
481
- config <<-CONFIG
482
- input {
483
- generator {
484
- count => #{event_count}
485
- message => "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
486
- }
487
- }
488
- filter {
489
- grok {
490
- match => { "message" => "%{SYSLOGLINE}" }
491
- overwrite => [ "message" ]
492
- }
493
- }
494
- output { null { } }
495
- CONFIG
496
-
497
- 2.times do
498
- start = Time.now
499
- agent do
500
- duration = (Time.now - start)
501
- puts "filters/grok parse rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s"
502
- insist { duration } < max_duration
503
- end
483
+ expect( event.get("foo-bar") ).to eql "hello"
504
484
  end
505
485
  end
506
486
 
507
487
  describe "single value match with duplicate-named fields in pattern" do
508
- config <<-CONFIG
509
- filter {
510
- grok {
511
- match => { "message" => "%{INT:foo}|%{WORD:foo}" }
512
- }
488
+ let(:config) {
489
+ {
490
+ 'match' => { "message" => "%{INT:foo}|%{WORD:foo}" }
513
491
  }
514
- CONFIG
492
+ }
515
493
 
516
494
  sample "hello world" do
517
- insist { subject.get("foo") }.is_a?(String)
495
+ expect( event.get("foo") ).to be_a(String)
518
496
  end
519
497
 
520
498
  sample "123 world" do
521
- insist { subject.get("foo") }.is_a?(String)
499
+ expect( event.get("foo") ).to be_a(String)
522
500
  end
523
501
  end
524
502
 
525
- describe "break_on_match default should be true and first match should exit filter" do
526
- config <<-CONFIG
527
- filter {
528
- grok {
529
- match => { "message" => "%{INT:foo}"
530
- "somefield" => "%{INT:bar}"}
531
- }
532
- }
533
- CONFIG
534
-
535
- sample("message" => "hello world 123", "somefield" => "testme abc 999") do
536
- insist { subject.get("foo") } == "123"
537
- insist { subject.get("bar") }.nil?
538
- end
539
- end
540
503
 
541
- describe "break_on_match when set to false should try all patterns" do
542
- config <<-CONFIG
543
- filter {
544
- grok {
545
- match => { "message" => "%{INT:foo}"
546
- "somefield" => "%{INT:bar}"}
547
- break_on_match => false
548
- }
504
+ describe "break_on_match default should be true" do
505
+ let(:config) {
506
+ {
507
+ 'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" }
549
508
  }
550
- CONFIG
509
+ }
510
+ let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
551
511
 
552
- sample("message" => "hello world 123", "somefield" => "testme abc 999") do
553
- insist { subject.get("foo") } == "123"
554
- insist { subject.get("bar") } == "999"
512
+ it 'exits filter after first match' do
513
+ expect( event.get("foo") ).to eql '123'
514
+ expect( event.get("bar") ).to be nil
555
515
  end
556
516
  end
557
517
 
558
- describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
559
- config <<-CONFIG
560
- filter {
561
- grok {
562
- match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
563
- break_on_match => false
564
- }
518
+ describe "break_on_match when set to false" do
519
+ let(:config) {
520
+ {
521
+ 'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" },
522
+ 'break_on_match' => false
565
523
  }
566
- CONFIG
567
-
568
- sample "treebranch" do
569
- insist { subject.get("name2") } == "branch"
570
- end
571
-
572
- sample "bushbeard" do
573
- insist { subject.get("name1") } == "bush"
574
- end
524
+ }
525
+ let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
575
526
 
576
- sample "treebeard" do
577
- insist { subject.get("name1") } == "tree"
578
- insist { subject.get("name2") } == "beard"
527
+ it 'should try all patterns' do
528
+ expect( event.get("foo") ).to eql '123'
529
+ expect( event.get("bar") ).to eql '999'
579
530
  end
580
531
  end
581
532
 
582
- describe "break_on_match default for array input with single grok pattern" do
583
- config <<-CONFIG
584
- filter {
585
- grok {
586
- match => { "message" => "%{INT:foo}"}
587
- }
533
+ context "break_on_match default for array input with single grok pattern" do
534
+ let(:config) {
535
+ {
536
+ 'match' => { "message" => "%{INT:foo}" },
537
+ 'break_on_match' => false
588
538
  }
589
- CONFIG
539
+ }
590
540
 
591
- # array input --
592
- sample("message" => ["hello world 123", "line 23"]) do
593
- insist { subject.get("foo") } == ["123", "23"]
594
- insist { subject.get("tags") }.nil?
541
+ describe 'fully matching input' do
542
+ let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
543
+ it 'matches' do
544
+ expect( event.get("foo") ).to eql ["123", "23"]
545
+ expect( event.get("tags") ).to be nil
546
+ end
595
547
  end
596
548
 
597
- # array input, one of them matches
598
- sample("message" => ["hello world 123", "abc"]) do
599
- insist { subject.get("foo") } == "123"
600
- insist { subject.get("tags") }.nil?
549
+ describe 'partially matching input' do
550
+ let(:data) { Hash("message" => ["hello world 123", "abc"]) } # array input, one of them matches
551
+ it 'matches' do
552
+ expect( event.get("foo") ).to eql "123"
553
+ expect( event.get("tags") ).to be nil
554
+ end
601
555
  end
602
556
  end
603
557
 
604
558
  describe "break_on_match = true (default) for array input with multiple grok pattern" do
605
- config <<-CONFIG
606
- filter {
607
- grok {
608
- match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
609
- }
559
+ let(:config) {
560
+ {
561
+ 'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
610
562
  }
611
- CONFIG
612
-
613
- # array input --
614
- sample("message" => ["hello world 123", "line 23"]) do
615
- insist { subject.get("foo") } == ["123", "23"]
616
- insist { subject.get("bar") }.nil?
617
- insist { subject.get("tags") }.nil?
563
+ }
564
+
565
+ describe 'matching input' do
566
+ let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
567
+ it 'matches' do
568
+ expect( event.get("foo") ).to eql ["123", "23"]
569
+ expect( event.get("bar") ).to be nil
570
+ expect( event.get("tags") ).to be nil
571
+ end
618
572
  end
619
573
 
620
- # array input, one of them matches
621
- sample("message" => ["hello world", "line 23"]) do
622
- insist { subject.get("bar") } == "hello"
623
- insist { subject.get("foo") } == "23"
624
- insist { subject.get("tags") }.nil?
574
+ describe 'partially matching input' do
575
+ let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
576
+ it 'matches' do
577
+ expect( event.get("bar") ).to eql 'hello'
578
+ expect( event.get("foo") ).to eql "23"
579
+ expect( event.get("tags") ).to be nil
580
+ end
625
581
  end
626
582
  end
627
583
 
628
584
  describe "break_on_match = false for array input with multiple grok pattern" do
629
- config <<-CONFIG
630
- filter {
631
- grok {
632
- match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
633
- break_on_match => false
634
- }
585
+ let(:config) {
586
+ {
587
+ 'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] },
588
+ 'break_on_match' => false
635
589
  }
636
- CONFIG
637
-
638
- # array input --
639
- sample("message" => ["hello world 123", "line 23"]) do
640
- insist { subject.get("foo") } == ["123", "23"]
641
- insist { subject.get("bar") } == ["hello", "line"]
642
- insist { subject.get("tags") }.nil?
590
+ }
591
+
592
+ describe 'fully matching input' do
593
+ let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
594
+ it 'matches' do
595
+ expect( event.get("foo") ).to eql ["123", "23"]
596
+ expect( event.get("bar") ).to eql ["hello", "line"]
597
+ expect( event.get("tags") ).to be nil
598
+ end
643
599
  end
644
600
 
645
- # array input, one of them matches
646
- sample("message" => ["hello world", "line 23"]) do
647
- insist { subject.get("bar") } == ["hello", "line"]
648
- insist { subject.get("foo") } == "23"
649
- insist { subject.get("tags") }.nil?
601
+ describe 'partially matching input' do
602
+ let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
603
+ it 'matches' do
604
+ expect( event.get("bar") ).to eql ["hello", "line"]
605
+ expect( event.get("foo") ).to eql "23"
606
+ expect( event.get("tags") ).to be nil
607
+ end
650
608
  end
651
609
  end
652
610
 
653
611
  describe "grok with unicode" do
654
- config <<-CONFIG
655
- filter {
656
- grok {
657
- #match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
658
- match => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
659
- }
612
+ let(:config) {
613
+ {
614
+ #'match' => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
615
+ 'match' => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
660
616
  }
661
- CONFIG
617
+ }
662
618
 
663
619
  sample "<22>Jan 4 07:50:46 mailmaster postfix/policy-spf[9454]: : SPF permerror (Junk encountered in record 'v=spf1 mx a:mail.domain.no ip4:192.168.0.4 �all'): Envelope-from: email@domain.no" do
664
- insist { subject.get("tags") }.nil?
665
- insist { subject.get("syslog_pri") } == "22"
666
- insist { subject.get("syslog_program") } == "postfix/policy-spf"
620
+ expect( event.get("tags") ).to be nil
621
+ expect( event.get("syslog_pri") ).to eql "22"
622
+ expect( event.get("syslog_program") ).to eql "postfix/policy-spf"
667
623
  end
668
624
  end
669
625
 
670
- describe "patterns in the 'patterns/' dir override core patterns" do
671
-
672
- let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
673
- let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
626
+ describe "grok with nil coerced value" do
627
+ let(:config) {
628
+ {
629
+ 'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
630
+ }
631
+ }
674
632
 
675
- before do
676
- FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
677
- @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
678
- @file.write('WORD \b[2-5]\b')
679
- @file.close
633
+ sample "test 28.4ms" do
634
+ expect( event.get("duration") ).to eql 28.4
635
+ expect( event.get("tags") ).to be nil
680
636
  end
681
637
 
682
- let(:config) do
683
- 'filter { grok { match => { "message" => "%{WORD:word}" } } }'
638
+ sample "test N/A" do
639
+ expect( event.to_hash ).not_to include("duration")
640
+ expect( event.get("tags") ).to be nil
684
641
  end
685
642
 
686
- sample("message" => 'hello') do
687
- insist { subject.get("tags") } == ["_grokparsefailure"]
643
+ sample "test abc" do
644
+ expect( event.get("duration") ).to be nil
645
+ expect( event.get("tags") ).to eql ["_grokparsefailure"]
688
646
  end
647
+ end
689
648
 
690
- after do
691
- File.unlink @file
692
- FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
649
+ describe "grok with nil coerced value and keep_empty_captures" do
650
+ let(:config) {
651
+ {
652
+ 'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" },
653
+ 'keep_empty_captures' => true
654
+ }
655
+ }
656
+
657
+ sample "test N/A" do
658
+ expect( event.to_hash ).to include("duration")
659
+ expect( event.get("tags") ).to be nil
693
660
  end
694
661
  end
695
662
 
696
- describe "patterns in custom dir override those in 'patterns/' dir" do
697
-
698
- let(:tmpdir) { Stud::Temporary.directory }
699
- let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
700
- let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
663
+ describe "grok with no coercion" do
664
+ let(:config) {
665
+ {
666
+ 'match' => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" },
667
+ }
668
+ }
701
669
 
702
- before do
703
- FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
704
- @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
705
- @file1.write('WORD \b[2-5]\b')
706
- @file1.close
707
- @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
708
- @file2.write('WORD \b[0-1]\b')
709
- @file2.close
670
+ sample "test 28.4ms" do
671
+ expect( event.get("duration") ).to eql '28.4'
672
+ expect( event.get("tags") ).to be nil
710
673
  end
711
674
 
712
- let(:config) do
713
- "filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
675
+ sample "test N/A" do
676
+ expect( event.get("duration") ).to be nil
677
+ expect( event.get("tags") ).to be nil
714
678
  end
679
+ end
715
680
 
716
- sample("message" => '0') do
717
- insist { subject.get("tags") } == nil
718
- end
681
+ describe "opening/closing" do
682
+ let(:config) { { "match" => {"message" => "A"} } }
683
+ let(:message) { 'AAA' }
719
684
 
720
- after do
721
- File.unlink @file1
722
- File.unlink @file2
723
- FileUtils.remove_entry tmpdir
724
- FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
685
+ it "should close cleanly" do
686
+ expect { subject.do_close }.not_to raise_error
725
687
  end
726
688
  end
727
689
 
728
- describe "patterns with file glob" do
729
-
730
- let(:tmpdir) { Stud::Temporary.directory }
690
+ describe "after grok when the event is JSON serialised the field values are unchanged" do
691
+ let(:config) {
692
+ {
693
+ 'match' => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"],
694
+ 'remove_field' => ["message","severity"],
695
+ 'add_tag' => ["ssh_failure"]
696
+ }
697
+ }
731
698
 
732
- before do
733
- @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
734
- @file3.write('WORD \b[0-1]\b')
735
- @file3.close
736
- @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
737
- @file4.write('WORD \b[2-5]\b')
738
- @file4.close
739
- end
699
+ sample('{"facility":"auth","message":"Failed password for testuser from 1.1.1.1 port 22"}') do
700
+ expect( event.get("username") ).to eql "testuser"
701
+ expect( event.get("port") ).to eql "22"
702
+ expect( event.get("src_ip") ).to eql "1.1.1.1"
703
+ expect( LogStash::Json.dump(event.get('username')) ).to eql "\"testuser\""
740
704
 
741
- let(:config) do
742
- "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
705
+ expect( event.to_json ).to match %r|"src_ip":"1.1.1.1"|
706
+ expect( event.to_json ).to match %r|"@timestamp":"#{Regexp.escape(event.get('@timestamp').to_s)}"|
707
+ expect( event.to_json ).to match %r|"port":"22"|
708
+ expect( event.to_json ).to match %r|"@version":"1"|
709
+ expect( event.to_json ).to match %r|"username"|i
710
+ expect( event.to_json ).to match %r|"testuser"|
711
+ expect( event.to_json ).to match %r|"tags":\["ssh_failure"\]|
743
712
  end
713
+ end
744
714
 
745
- sample("message" => '0') do
746
- insist { subject.get("tags") } == nil
747
- end
715
+ describe "grok with inline pattern definition successfully extracts fields" do
716
+ let(:config) {
717
+ {
718
+ 'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" },
719
+ 'pattern_definitions' => {
720
+ "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
721
+ "MY_PATTERN" => "%{YEAR}"
722
+ }
723
+ }
724
+ }
748
725
 
749
- after do
750
- File.unlink @file3
751
- File.unlink @file4
752
- FileUtils.remove_entry tmpdir
726
+ sample "Mon Dec 26 16:22:08 2016 error 2020" do
727
+ expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
728
+ expect( event.get("level") ).to eql "error"
729
+ expect( event.get("hindsight") ).to eql "2020"
753
730
  end
754
731
  end
755
732
 
756
- describe "patterns with file glob on directory that contains subdirectories" do
757
-
758
- let(:tmpdir) { Stud::Temporary.directory }
733
+ describe "grok with inline pattern definition overwrites existing pattern definition" do
734
+ let(:config) {
735
+ {
736
+ 'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" },
737
+ # loglevel was previously ([Aa]lert|ALERT|[Tt]...
738
+ 'pattern_definitions' => {
739
+ "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
740
+ "LOGLEVEL" => "%{NUMBER}"
741
+ }
742
+ }
743
+ }
759
744
 
760
- before do
761
- @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
762
- @file3.write('WORD \b[0-1]\b')
763
- @file3.close
764
- Dir.mkdir(File.join(tmpdir, "subdir"))
745
+ sample "Mon Dec 26 16:22:08 2016 9999" do
746
+ expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
747
+ expect( event.get("level") ).to eql "9999"
765
748
  end
749
+ end
766
750
 
751
+ context 'when timeouts are explicitly disabled' do
767
752
  let(:config) do
768
- "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
753
+ {
754
+ "timeout_millis" => 0
755
+ }
769
756
  end
770
757
 
771
- sample("message" => '0') do
772
- insist { subject.get("tags") } == nil
773
- end
758
+ context 'when given a pathological input', slow: true do
759
+ let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
760
+ let(:config) { super().merge("match" => { "message" => "(.*a){30}" }) }
774
761
 
775
- after do
776
- File.unlink @file3
777
- FileUtils.remove_entry tmpdir
762
+ it 'blocks for at least 3 seconds' do
763
+ blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
764
+ expect do
765
+ Timeout.timeout(3, blocking_exception_class) do
766
+ subject.filter(event)
767
+ end
768
+ end.to raise_exception(blocking_exception_class)
769
+ end
778
770
  end
779
771
  end
772
+ end
780
773
 
781
- describe "grok with nil coerced value" do
782
- config <<-CONFIG
783
- filter {
784
- grok {
785
- match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
786
- }
787
- }
788
- CONFIG
774
+ describe LogStash::Filters::Grok do
789
775
 
790
- sample "test 28.4ms" do
791
- insist { subject.get("duration") } == 28.4
792
- insist { subject.get("tags") }.nil?
793
- end
776
+ subject(:grok_filter) { described_class.new(config) }
777
+ let(:config) { {} }
794
778
 
795
- sample "test N/A" do
796
- insist { insist { subject.to_hash }.include?("duration") }.fails
797
- insist { subject.get("tags") }.nil?
798
- end
779
+ context 'when initialized with `ecs_compatibility => v8`' do
780
+ let(:config) { super().merge("ecs_compatibility" => "v8", "match" => ["message", "%{SYSLOGLINE}"]) }
781
+ context '#register' do
782
+ let(:logger_stub) { double('Logger').as_null_object }
783
+ before(:each) { allow_any_instance_of(described_class).to receive(:logger).and_return(logger_stub)}
799
784
 
800
- sample "test abc" do
801
- insist { subject.get("duration") }.nil?
802
- insist { subject.get("tags") } == ["_grokparsefailure"]
785
+ it 'logs a helpful warning about the unreleased v8' do
786
+ grok_filter.register
787
+
788
+ expect(logger_stub).to have_received(:warn).with(a_string_including "preview of the unreleased ECS v8")
789
+ end
803
790
  end
804
791
  end
792
+ end
805
793
 
806
- describe "grok with nil coerced value and keep_empty_captures" do
807
- config <<-CONFIG
808
- filter {
809
- grok {
810
- match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
811
- keep_empty_captures => true
812
- }
813
- }
814
- CONFIG
794
+ describe LogStash::Filters::Grok do
795
+ describe "(LEGACY)" do
796
+ describe "patterns in the 'patterns/' dir override core patterns" do
815
797
 
816
- sample "test N/A" do
817
- insist { subject.to_hash }.include?("duration")
818
- insist { subject.get("tags") }.nil?
819
- end
798
+ let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
799
+ let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
820
800
 
821
- end
801
+ before do
802
+ FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
803
+ @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
804
+ @file.write('WORD \b[2-5]\b')
805
+ @file.close
806
+ end
822
807
 
823
- describe "grok with no coercion" do
824
- config <<-CONFIG
825
- filter {
826
- grok {
827
- match => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" }
828
- }
829
- }
830
- CONFIG
808
+ let(:config) do
809
+ 'filter { grok { match => { "message" => "%{WORD:word}" } } }'
810
+ end
831
811
 
832
- sample "test 28.4ms" do
833
- insist { subject.get("duration") } == "28.4"
834
- insist { subject.get("tags") }.nil?
835
- end
812
+ sample("message" => 'hello') do
813
+ expect(subject.get("tags")).to eql ["_grokparsefailure"]
814
+ end
836
815
 
837
- sample "test N/A" do
838
- insist { subject.get("duration") }.nil?
839
- insist { subject.get("tags") }.nil?
816
+ after do
817
+ File.unlink @file
818
+ FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
819
+ end
840
820
  end
841
- end
842
821
 
843
- describe "opening/closing" do
844
- let(:config) { {"match" => {"message" => "A"}} }
845
- subject(:plugin) do
846
- ::LogStash::Filters::Grok.new(config)
847
- end
822
+ describe "patterns in custom dir override those in 'patterns/' dir" do
848
823
 
849
- before do
850
- plugin.register
851
- end
824
+ let(:tmpdir) { Stud::Temporary.directory }
825
+ let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
826
+ let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
852
827
 
853
- it "should close cleanly" do
854
- expect { plugin.do_close }.not_to raise_error
855
- end
856
- end
828
+ before do
829
+ FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
830
+ @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
831
+ @file1.write('WORD \b[2-5]\b')
832
+ @file1.close
833
+ @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
834
+ @file2.write('WORD \b[0-1]\b')
835
+ @file2.close
836
+ end
857
837
 
858
- describe "after grok when the event is JSON serialised the field values are unchanged" do
859
- config <<-CONFIG
860
- filter {grok {match => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"] remove_field => ["message","severity"] add_tag => ["ssh_failure"]}}
861
- CONFIG
838
+ let(:config) do
839
+ "filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
840
+ end
862
841
 
863
- sample('{"facility":"auth","message":"Failed password for testuser from 1.1.1.1 port 22"}') do
864
- insist { subject.get("username") } == "testuser"
865
- insist { subject.get("port") } == "22"
866
- insist { subject.get("src_ip") } == "1.1.1.1"
867
- insist { LogStash::Json.dump(subject.get('username')) } == "\"testuser\""
842
+ sample("message" => '0') do
843
+ expect(subject.get("tags")).to be nil
844
+ end
868
845
 
869
- insist { subject.to_json } =~ %r|"src_ip":"1.1.1.1"|
870
- insist { subject.to_json } =~ %r|"@timestamp":"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ"|
871
- insist { subject.to_json } =~ %r|"port":"22"|
872
- insist { subject.to_json } =~ %r|"@version":"1"|
873
- insist { subject.to_json } =~ %r|"username"|i
874
- insist { subject.to_json } =~ %r|"testuser"|
875
- insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]|
846
+ after do
847
+ File.unlink @file1
848
+ File.unlink @file2
849
+ FileUtils.remove_entry tmpdir
850
+ FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
851
+ end
876
852
  end
877
- end
878
853
 
879
- describe "grok with inline pattern definition successfully extracts fields" do
880
- config <<-CONFIG
881
- filter {
882
- grok {
883
- match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" }
884
- pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
885
- "MY_PATTERN" => "%{YEAR}"}
886
- }
887
- }
888
- CONFIG
854
+ describe "patterns with file glob" do
889
855
 
890
- sample "Mon Dec 26 16:22:08 2016 error 2020" do
891
- insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
892
- insist { subject.get("level") } == "error"
893
- insist { subject.get("hindsight") } == "2020"
894
- end
895
- end
856
+ let(:tmpdir) { Stud::Temporary.directory }
896
857
 
897
- describe "grok with inline pattern definition overwrites existing pattern definition" do
898
- config <<-CONFIG
899
- filter {
900
- grok {
901
- match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" }
902
- # loglevel was previously ([Aa]lert|ALERT|[Tt]...
903
- pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
904
- "LOGLEVEL" => "%{NUMBER}"}
905
- }
906
- }
907
- CONFIG
858
+ before do
859
+ @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
860
+ @file3.write('WORD \b[0-1]\b')
861
+ @file3.close
862
+ @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
863
+ @file4.write('WORD \b[2-5]\b')
864
+ @file4.close
865
+ end
908
866
 
909
- sample "Mon Dec 26 16:22:08 2016 9999" do
910
- insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
911
- insist { subject.get("level") } == "9999"
912
- end
913
- end
867
+ let(:config) do
868
+ "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
869
+ end
914
870
 
871
+ sample("message" => '0') do
872
+ expect(subject.get("tags")).to be nil
873
+ end
915
874
 
916
- describe "direct plugin testing" do
917
- subject do
918
- plugin = LogStash::Filters::Grok.new(options)
919
- plugin.register
920
- plugin
875
+ after do
876
+ File.unlink @file3
877
+ File.unlink @file4
878
+ FileUtils.remove_entry tmpdir
879
+ end
921
880
  end
922
881
 
923
- let(:data) { {"message" => message} }
924
- let(:event) { LogStash::Event.new(data) }
882
+ describe "patterns with file glob on directory that contains subdirectories" do
925
883
 
926
- context 'when timeouts are explicitly disabled' do
927
- let(:options) do
928
- {
929
- "timeout_millis" => 0
884
+ let(:tmpdir) { Stud::Temporary.directory }
885
+
886
+ before do
887
+ @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
888
+ @file3.write('WORD \b[0-1]\b')
889
+ @file3.close
890
+ Dir.mkdir(File.join(tmpdir, "subdir"))
891
+ end
892
+
893
+ let(:config) do
894
+ "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
895
+ end
896
+
897
+ sample("message" => '0') do
898
+ expect(subject.get("tags")).to be nil
899
+ end
900
+
901
+ after do
902
+ File.unlink @file3
903
+ FileUtils.remove_entry tmpdir
904
+ end
905
+ end
906
+
907
+ describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
908
+ config <<-CONFIG
909
+ filter {
910
+ grok {
911
+ match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
912
+ break_on_match => false
930
913
  }
914
+ }
915
+ CONFIG
916
+
917
+ sample "treebranch" do
918
+ expect(subject.get("name2")).to eql "branch"
931
919
  end
932
920
 
933
- context 'when given a pathological input' do
934
- let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
935
- let(:options) { super().merge("match" => { "message" => "(.*a){30}" }) }
921
+ sample "bushbeard" do
922
+ expect(subject.get("name1")).to eql "bush"
923
+ end
936
924
 
937
- it 'blocks for at least 3 seconds' do
938
- blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
939
- expect do
940
- Timeout.timeout(3, blocking_exception_class) do
941
- subject.filter(event)
942
- end
943
- end.to raise_exception(blocking_exception_class)
944
- end
925
+ sample "treebeard" do
926
+ expect(subject.get("name1")).to eql "tree"
927
+ expect(subject.get("name2")).to eql "beard"
945
928
  end
946
929
  end
947
930
  end
948
- end
931
+ end