logstash-filter-grok 4.1.1 → 4.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/Gemfile +6 -0
- data/LICENSE +199 -10
- data/README.md +1 -1
- data/docs/index.asciidoc +66 -0
- data/lib/logstash/filters/grok.rb +158 -46
- data/logstash-filter-grok.gemspec +4 -4
- data/spec/filters/grok_performance_spec.rb +144 -0
- data/spec/filters/grok_spec.rb +629 -646
- data/spec/spec_helper.rb +19 -0
- metadata +33 -10
data/spec/filters/grok_spec.rb
CHANGED
@@ -1,392 +1,369 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
|
3
|
-
require "stud/temporary"
|
2
|
+
require_relative "../spec_helper"
|
4
3
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
4
|
+
require "logstash/filters/grok"
|
5
|
+
|
6
|
+
describe LogStash::Filters::Grok do
|
7
|
+
subject { described_class.new(config) }
|
8
|
+
let(:config) { {} }
|
9
|
+
let(:event) { LogStash::Event.new(data) }
|
10
|
+
let(:data) { { "message" => message } }
|
11
|
+
|
12
|
+
before(:each) do
|
13
|
+
subject.register
|
14
|
+
subject.filter(event)
|
11
15
|
end
|
12
16
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
+
def self.sample(message, &block)
|
18
|
+
# mod = RSpec::Core::MemoizedHelpers.module_for(self)
|
19
|
+
# mod.attr_reader :message
|
20
|
+
# # mod.__send__(:define_method, :message) { message }
|
21
|
+
# it("matches: #{message}") { @message = message; block.call }
|
22
|
+
describe message do
|
23
|
+
let(:message) { message }
|
24
|
+
it("groks", &block)
|
17
25
|
end
|
18
26
|
end
|
19
|
-
end
|
20
27
|
|
21
|
-
|
28
|
+
describe "simple syslog line" do
|
29
|
+
let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "overwrite" => [ "message" ] } }
|
30
|
+
let(:message) { 'Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]' }
|
31
|
+
|
32
|
+
it "matches pattern" do
|
33
|
+
expect( event.get("tags") ).to be nil
|
34
|
+
expect( event.get("logsource") ).to eql "evita"
|
35
|
+
expect( event.get("timestamp") ).to eql "Mar 16 00:01:25"
|
36
|
+
expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
|
37
|
+
expect( event.get("program") ).to eql "postfix/smtpd"
|
38
|
+
expect( event.get("pid") ).to eql "1713"
|
39
|
+
end
|
40
|
+
|
41
|
+
%w(v1 v8).each do |ecs_mode|
|
42
|
+
context "in ecs mode #{ecs_mode}" do
|
43
|
+
let(:config) { super().merge('ecs_compatibility' => ecs_mode) }
|
44
|
+
|
45
|
+
it "matches pattern" do
|
46
|
+
expect( event.get("host") ).to eql "hostname"=>"evita"
|
47
|
+
expect( event.get("process") ).to eql "name"=>"postfix/smtpd", "pid"=>1713
|
48
|
+
expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
22
52
|
|
23
|
-
|
53
|
+
context 'with target' do
|
54
|
+
let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "grok" } }
|
24
55
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
grok
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
}
|
35
|
-
CONFIG
|
56
|
+
it "matches pattern" do
|
57
|
+
expect( event.get("message") ).to eql message
|
58
|
+
expect( event.get("tags") ).to be nil
|
59
|
+
expect( event.get("grok") ).to_not be nil
|
60
|
+
expect( event.get("[grok][timestamp]") ).to eql "Mar 16 00:01:25"
|
61
|
+
expect( event.get("[grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
|
62
|
+
expect( event.get("[grok][pid]") ).to eql "1713"
|
63
|
+
end
|
64
|
+
end
|
36
65
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
66
|
+
context 'with [deep] target' do
|
67
|
+
let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "[@metadata][grok]" } }
|
68
|
+
|
69
|
+
it "matches pattern" do
|
70
|
+
expect( event.get("message") ).to eql message
|
71
|
+
expect( event.get("tags") ).to be nil
|
72
|
+
expect( event.get("grok") ).to be nil
|
73
|
+
expect( event.get("[@metadata][grok][logsource]") ).to eql "evita"
|
74
|
+
expect( event.get("[@metadata][grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
|
75
|
+
end
|
44
76
|
end
|
45
77
|
end
|
46
78
|
|
47
79
|
describe "ietf 5424 syslog line" do
|
48
|
-
|
49
|
-
# At this time, only filters are supported.
|
50
|
-
config <<-CONFIG
|
51
|
-
filter {
|
52
|
-
grok {
|
53
|
-
match => { "message" => "%{SYSLOG5424LINE}" }
|
54
|
-
}
|
55
|
-
}
|
56
|
-
CONFIG
|
80
|
+
let(:config) { { "match" => { "message" => "%{SYSLOG5424LINE}" } } }
|
57
81
|
|
58
82
|
sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\"bar\"][id2 baz=\"something\"] Hello, syslog." do
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
83
|
+
expect( event.get("tags") ).to be nil
|
84
|
+
expect( event.get("syslog5424_pri") ).to eql "191"
|
85
|
+
expect( event.get("syslog5424_ver") ).to eql "1"
|
86
|
+
expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
|
87
|
+
expect( event.get("syslog5424_host") ).to eql "paxton.local"
|
88
|
+
expect( event.get("syslog5424_app") ).to eql "grokdebug"
|
89
|
+
expect( event.get("syslog5424_proc") ).to eql "4123"
|
90
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
91
|
+
expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"][id2 baz=\"something\"]"
|
92
|
+
expect( event.get("syslog5424_msg") ).to eql "Hello, syslog."
|
69
93
|
end
|
70
94
|
|
71
95
|
sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - [id1 foo=\"bar\"] No process ID." do
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
96
|
+
expect( event.get("tags") ).to be nil
|
97
|
+
expect( event.get("syslog5424_pri") ).to eql "191"
|
98
|
+
expect( event.get("syslog5424_ver") ).to eql "1"
|
99
|
+
expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
|
100
|
+
expect( event.get("syslog5424_host") ).to eql "paxton.local"
|
101
|
+
expect( event.get("syslog5424_app") ).to eql "grokdebug"
|
102
|
+
expect( event.get("syslog5424_proc") ).to be nil
|
103
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
104
|
+
expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"]"
|
105
|
+
expect( event.get("syslog5424_msg") ).to eql "No process ID."
|
82
106
|
end
|
83
107
|
|
84
108
|
sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - - No structured data." do
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
109
|
+
expect( event.get("tags") ).to be nil
|
110
|
+
expect( event.get("syslog5424_pri") ).to eql "191"
|
111
|
+
expect( event.get("syslog5424_ver") ).to eql "1"
|
112
|
+
expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
|
113
|
+
expect( event.get("syslog5424_host") ).to eql "paxton.local"
|
114
|
+
expect( event.get("syslog5424_app") ).to eql "grokdebug"
|
115
|
+
expect( event.get("syslog5424_proc") ).to eql '4123'
|
116
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
117
|
+
expect( event.get("syslog5424_sd") ).to be nil
|
118
|
+
expect( event.get("syslog5424_msg") ).to eql "No structured data."
|
95
119
|
end
|
96
120
|
|
97
121
|
sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - - No PID or SD." do
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
122
|
+
expect( event.get("tags") ).to be nil
|
123
|
+
expect( event.get("syslog5424_pri") ).to eql "191"
|
124
|
+
expect( event.get("syslog5424_ver") ).to eql "1"
|
125
|
+
expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
|
126
|
+
expect( event.get("syslog5424_host") ).to eql "paxton.local"
|
127
|
+
expect( event.get("syslog5424_app") ).to eql "grokdebug"
|
128
|
+
expect( event.get("syslog5424_proc") ).to be nil
|
129
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
130
|
+
expect( event.get("syslog5424_sd") ).to be nil
|
131
|
+
expect( event.get("syslog5424_msg") ).to eql "No PID or SD."
|
108
132
|
end
|
109
133
|
|
110
134
|
sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - Missing structured data." do
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
insist { subject.get("syslog5424_proc") } == "4123"
|
118
|
-
insist { subject.get("syslog5424_msgid") } == nil
|
119
|
-
insist { subject.get("syslog5424_sd") } == nil
|
120
|
-
insist { subject.get("syslog5424_msg") } == "Missing structured data."
|
135
|
+
expect( event.get("tags") ).to be nil
|
136
|
+
|
137
|
+
expect( event.get("syslog5424_proc") ).to eql '4123'
|
138
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
139
|
+
expect( event.get("syslog5424_sd") ).to be nil
|
140
|
+
expect( event.get("syslog5424_msg") ).to eql "Missing structured data."
|
121
141
|
end
|
122
142
|
|
123
143
|
sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - - Additional spaces." do
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
insist { subject.get("syslog5424_msgid") } == nil
|
132
|
-
insist { subject.get("syslog5424_sd") } == nil
|
133
|
-
insist { subject.get("syslog5424_msg") } == "Additional spaces."
|
144
|
+
expect( event.get("tags") ).to be nil
|
145
|
+
|
146
|
+
expect( event.get("syslog5424_app") ).to eql "grokdebug"
|
147
|
+
expect( event.get("syslog5424_proc") ).to eql '4123'
|
148
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
149
|
+
expect( event.get("syslog5424_sd") ).to be nil
|
150
|
+
expect( event.get("syslog5424_msg") ).to eql "Additional spaces."
|
134
151
|
end
|
135
152
|
|
136
153
|
sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - Additional spaces and missing SD." do
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
insist { subject.get("syslog5424_msgid") } == nil
|
145
|
-
insist { subject.get("syslog5424_sd") } == nil
|
146
|
-
insist { subject.get("syslog5424_msg") } == "Additional spaces and missing SD."
|
154
|
+
expect( event.get("tags") ).to be nil
|
155
|
+
|
156
|
+
expect( event.get("syslog5424_app") ).to eql "grokdebug"
|
157
|
+
expect( event.get("syslog5424_proc") ).to eql '4123'
|
158
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
159
|
+
expect( event.get("syslog5424_sd") ).to be nil
|
160
|
+
expect( event.get("syslog5424_msg") ).to eql "Additional spaces and missing SD."
|
147
161
|
end
|
148
162
|
|
149
163
|
sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 dnsmasq-dhcp 8048 - - Appname contains a dash" do
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
164
|
+
expect( event.get("tags") ).to be nil
|
165
|
+
expect( event.get("syslog5424_pri") ).to eql "30"
|
166
|
+
expect( event.get("syslog5424_ver") ).to eql "1"
|
167
|
+
expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
|
168
|
+
expect( event.get("syslog5424_host") ).to eql "osctrl01"
|
169
|
+
expect( event.get("syslog5424_app") ).to eql "dnsmasq-dhcp"
|
170
|
+
expect( event.get("syslog5424_proc") ).to eql "8048"
|
171
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
172
|
+
expect( event.get("syslog5424_sd") ).to be nil
|
173
|
+
expect( event.get("syslog5424_msg") ).to eql "Appname contains a dash"
|
160
174
|
end
|
161
175
|
|
162
176
|
sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 - 8048 - - Appname is nil" do
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
177
|
+
expect( event.get("tags") ).to be nil
|
178
|
+
expect( event.get("syslog5424_pri") ).to eql "30"
|
179
|
+
expect( event.get("syslog5424_ver") ).to eql "1"
|
180
|
+
expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
|
181
|
+
expect( event.get("syslog5424_host") ).to eql "osctrl01"
|
182
|
+
expect( event.get("syslog5424_app") ).to be nil
|
183
|
+
expect( event.get("syslog5424_proc") ).to eql "8048"
|
184
|
+
expect( event.get("syslog5424_msgid") ).to be nil
|
185
|
+
expect( event.get("syslog5424_sd") ).to be nil
|
186
|
+
expect( event.get("syslog5424_msg") ).to eql "Appname is nil"
|
173
187
|
end
|
174
188
|
end
|
175
189
|
|
176
|
-
describe "parsing an event with multiple messages (array of strings)", :
|
177
|
-
config
|
178
|
-
|
179
|
-
grok {
|
180
|
-
match => { "message" => "(?:hello|world) %{NUMBER}" }
|
181
|
-
named_captures_only => false
|
182
|
-
}
|
183
|
-
}
|
184
|
-
CONFIG
|
190
|
+
describe "parsing an event with multiple messages (array of strings)", if: false do
|
191
|
+
let(:config) { { "message" => "(?:hello|world) %{NUMBER}" } }
|
192
|
+
let(:message) { [ "hello 12345", "world 23456" ] }
|
185
193
|
|
186
|
-
|
187
|
-
|
194
|
+
it "matches them all" do
|
195
|
+
expect( event.get("NUMBER") ).to eql [ "12345", "23456" ]
|
188
196
|
end
|
189
197
|
end
|
190
198
|
|
191
199
|
describe "coercing matched values" do
|
192
|
-
config
|
193
|
-
|
194
|
-
grok {
|
195
|
-
match => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" }
|
196
|
-
}
|
197
|
-
}
|
198
|
-
CONFIG
|
200
|
+
let(:config) { { "match" => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" } } }
|
201
|
+
let(:message) { '400 454.33' }
|
199
202
|
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
203
|
+
it "coerces matched values" do
|
204
|
+
expect( event.get("foo") ).to be_a Integer
|
205
|
+
expect( event.get("foo") ).to eql 400
|
206
|
+
expect( event.get("bar") ).to be_a Float
|
207
|
+
expect( event.get("bar") ).to eql 454.33
|
205
208
|
end
|
206
209
|
end
|
207
210
|
|
208
211
|
describe "in-line pattern definitions" do
|
209
|
-
config
|
210
|
-
filter {
|
211
|
-
grok {
|
212
|
-
match => { "message" => "%{FIZZLE=\\d+}" }
|
213
|
-
named_captures_only => false
|
214
|
-
}
|
215
|
-
}
|
216
|
-
CONFIG
|
212
|
+
let(:config) { { "match" => { "message" => "%{FIZZLE=\\d+}" }, "named_captures_only" => false } }
|
217
213
|
|
218
214
|
sample "hello 1234" do
|
219
|
-
|
215
|
+
expect( event.get("FIZZLE") ).to eql '1234'
|
220
216
|
end
|
221
217
|
end
|
222
218
|
|
223
219
|
describe "processing selected fields" do
|
224
|
-
config
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
match => { "examplefield" => "%{NUMBER:num}" }
|
229
|
-
break_on_match => false
|
230
|
-
}
|
220
|
+
let(:config) {
|
221
|
+
{
|
222
|
+
'match' => { "message" => "%{WORD:word}", "examplefield" => "%{NUMBER:num}" },
|
223
|
+
'break_on_match' => false
|
231
224
|
}
|
232
|
-
|
225
|
+
}
|
226
|
+
let(:data) { { "message" => "hello world", "examplefield" => "12345" } }
|
233
227
|
|
234
|
-
|
235
|
-
|
236
|
-
|
228
|
+
it "processes declared matches" do
|
229
|
+
expect( event.get("word") ).to eql 'hello'
|
230
|
+
expect( event.get("examplefield") ).to eql '12345'
|
237
231
|
end
|
238
232
|
end
|
239
233
|
|
240
234
|
describe "adding fields on match" do
|
241
|
-
config
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
add_field => [ "new_field", "%{fancy}" ]
|
246
|
-
}
|
235
|
+
let(:config) {
|
236
|
+
{
|
237
|
+
'match' => { "message" => "matchme %{NUMBER:fancy}" },
|
238
|
+
'add_field' => [ "new_field", "%{fancy}" ]
|
247
239
|
}
|
248
|
-
|
240
|
+
}
|
249
241
|
|
250
242
|
sample "matchme 1234" do
|
251
|
-
|
252
|
-
|
243
|
+
expect( event.get("tags") ).to be nil
|
244
|
+
expect( event.get("new_field") ).to eql "1234"
|
253
245
|
end
|
254
246
|
|
255
247
|
sample "this will not be matched" do
|
256
|
-
|
257
|
-
|
248
|
+
expect( event.get("tags") ).to include("_grokparsefailure")
|
249
|
+
expect( event ).not_to include 'new_field'
|
258
250
|
end
|
259
251
|
end
|
260
252
|
|
261
253
|
context "empty fields" do
|
262
254
|
describe "drop by default" do
|
263
|
-
config
|
264
|
-
|
265
|
-
|
266
|
-
match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
|
267
|
-
}
|
255
|
+
let(:config) {
|
256
|
+
{
|
257
|
+
'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
|
268
258
|
}
|
269
|
-
|
259
|
+
}
|
270
260
|
|
271
261
|
sample "1=test" do
|
272
|
-
|
273
|
-
|
262
|
+
expect( event.get("tags") ).to be nil
|
263
|
+
expect( event ).to include 'foo1'
|
274
264
|
|
275
265
|
# Since 'foo2' was not captured, it must not be present in the event.
|
276
|
-
|
266
|
+
expect( event ).not_to include 'foo2'
|
277
267
|
end
|
278
268
|
end
|
279
269
|
|
280
270
|
describe "keep if keep_empty_captures is true" do
|
281
|
-
config
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
keep_empty_captures => true
|
286
|
-
}
|
271
|
+
let(:config) {
|
272
|
+
{
|
273
|
+
'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" },
|
274
|
+
'keep_empty_captures' => true
|
287
275
|
}
|
288
|
-
|
276
|
+
}
|
289
277
|
|
290
278
|
sample "1=test" do
|
291
|
-
|
279
|
+
expect( event.get("tags") ).to be nil
|
292
280
|
# use .to_hash for this test, for now, because right now
|
293
281
|
# the Event.include? returns false for missing fields as well
|
294
282
|
# as for fields with nil values.
|
295
|
-
|
296
|
-
|
283
|
+
expect( event.to_hash ).to include 'foo1'
|
284
|
+
expect( event.to_hash ).to include 'foo2'
|
297
285
|
end
|
298
286
|
end
|
299
287
|
end
|
300
288
|
|
301
289
|
describe "when named_captures_only == false" do
|
302
|
-
config
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
named_captures_only => false
|
307
|
-
}
|
290
|
+
let(:config) {
|
291
|
+
{
|
292
|
+
'match' => { "message" => "Hello %{WORD}. %{WORD:foo}" },
|
293
|
+
'named_captures_only' => false
|
308
294
|
}
|
309
|
-
|
295
|
+
}
|
310
296
|
|
311
297
|
sample "Hello World, yo!" do
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
298
|
+
expect( event ).to include 'WORD'
|
299
|
+
expect( event.get("WORD") ).to eql "World"
|
300
|
+
expect( event ).to include 'foo'
|
301
|
+
expect( event.get("foo") ).to eql "yo"
|
316
302
|
end
|
317
303
|
end
|
318
304
|
|
319
305
|
describe "using oniguruma named captures (?<name>regex)" do
|
320
306
|
context "plain regexp" do
|
321
|
-
config
|
322
|
-
|
323
|
-
|
324
|
-
match => { "message" => "(?<foo>\w+)" }
|
325
|
-
}
|
307
|
+
let(:config) {
|
308
|
+
{
|
309
|
+
'match' => { "message" => "(?<foo>\\w+)" }
|
326
310
|
}
|
327
|
-
|
311
|
+
}
|
312
|
+
|
328
313
|
sample "hello world" do
|
329
|
-
|
330
|
-
|
314
|
+
expect( event.get("tags") ).to be nil
|
315
|
+
expect( event.get("foo") ).to eql "hello"
|
331
316
|
end
|
332
317
|
end
|
333
318
|
|
334
319
|
context "grok patterns" do
|
335
|
-
config
|
336
|
-
|
337
|
-
|
338
|
-
match => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
|
339
|
-
}
|
320
|
+
let(:config) {
|
321
|
+
{
|
322
|
+
'match' => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
|
340
323
|
}
|
341
|
-
|
324
|
+
}
|
342
325
|
|
343
326
|
sample "fancy 12-12-12 12:12:12" do
|
344
|
-
|
345
|
-
|
327
|
+
expect( event.get("tags") ).to be nil
|
328
|
+
expect( event.get("timestamp") ).to eql "12-12-12 12:12:12"
|
346
329
|
end
|
347
330
|
end
|
348
331
|
end
|
349
332
|
|
350
333
|
describe "grok on integer types" do
|
351
|
-
config
|
352
|
-
|
353
|
-
|
354
|
-
match => { "status" => "^403$" }
|
355
|
-
add_tag => "four_oh_three"
|
356
|
-
}
|
334
|
+
let(:config) {
|
335
|
+
{
|
336
|
+
'match' => { "status" => "^403$" }, 'add_tag' => "four_oh_three"
|
357
337
|
}
|
358
|
-
|
338
|
+
}
|
339
|
+
let(:data) { Hash({ "status" => 403 }) }
|
359
340
|
|
360
|
-
|
361
|
-
|
362
|
-
|
341
|
+
it "parses" do
|
342
|
+
expect( event.get("tags") ).not_to include "_grokparsefailure"
|
343
|
+
expect( event.get("tags") ).to include "four_oh_three"
|
363
344
|
end
|
364
345
|
end
|
365
346
|
|
366
347
|
describe "grok on float types" do
|
367
|
-
config
|
368
|
-
|
369
|
-
|
370
|
-
match => { "version" => "^1.0$" }
|
371
|
-
add_tag => "one_point_oh"
|
372
|
-
}
|
348
|
+
let(:config) {
|
349
|
+
{
|
350
|
+
'match' => { "version" => "^1.0$" }, 'add_tag' => "one_point_oh"
|
373
351
|
}
|
374
|
-
|
352
|
+
}
|
353
|
+
let(:data) { Hash({ "version" => 1.0 }) }
|
375
354
|
|
376
|
-
|
377
|
-
|
378
|
-
|
355
|
+
it "parses" do
|
356
|
+
expect( event.get("tags") ).not_to include "_grokparsefailure"
|
357
|
+
expect( event.get("tags") ).to include "one_point_oh"
|
379
358
|
end
|
380
359
|
end
|
381
360
|
|
382
361
|
describe "grok on %{LOGLEVEL}" do
|
383
|
-
config
|
384
|
-
|
385
|
-
|
386
|
-
match => { "message" => "%{LOGLEVEL:level}: error!" }
|
387
|
-
}
|
362
|
+
let(:config) {
|
363
|
+
{
|
364
|
+
'match' => { "message" => "%{LOGLEVEL:level}: error!" }
|
388
365
|
}
|
389
|
-
|
366
|
+
}
|
390
367
|
|
391
368
|
log_level_names = %w(
|
392
369
|
trace Trace TRACE
|
@@ -402,547 +379,553 @@ describe LogStash::Filters::Grok do
|
|
402
379
|
)
|
403
380
|
log_level_names.each do |level_name|
|
404
381
|
sample "#{level_name}: error!" do
|
405
|
-
|
382
|
+
expect( event.get("level") ).to eql level_name
|
406
383
|
end
|
407
384
|
end
|
408
385
|
end
|
409
386
|
|
410
387
|
describe "timeout on failure" do
|
411
|
-
config
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
message => "(.*a){30}"
|
416
|
-
}
|
417
|
-
timeout_millis => 100
|
418
|
-
}
|
388
|
+
let(:config) {
|
389
|
+
{
|
390
|
+
'match' => { "message" => "(.*a){30}" },
|
391
|
+
'timeout_millis' => 100
|
419
392
|
}
|
420
|
-
|
393
|
+
}
|
421
394
|
|
422
395
|
sample "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" do
|
423
|
-
expect(
|
424
|
-
expect(
|
396
|
+
expect( event.get("tags") ).to include("_groktimeout")
|
397
|
+
expect( event.get("tags") ).not_to include("_grokparsefailure")
|
398
|
+
end
|
399
|
+
end
|
400
|
+
|
401
|
+
describe "no timeout on failure with multiple patterns (when timeout not grouped)" do
|
402
|
+
let(:config) {
|
403
|
+
{
|
404
|
+
'match' => {
|
405
|
+
"message" => [
|
406
|
+
"(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
|
407
|
+
"(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
|
408
|
+
"(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
|
409
|
+
"(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
|
410
|
+
"(.*a){20}"
|
411
|
+
]
|
412
|
+
},
|
413
|
+
'timeout_millis' => 750,
|
414
|
+
'timeout_scope' => 'pattern'
|
415
|
+
}
|
416
|
+
}
|
417
|
+
|
418
|
+
sample( 'b' * 15 + 'c' * 15 + 'd' * 15 + 'e' * 15 + ' ' + 'a' * 20 ) do
|
419
|
+
expect( event.get("tags") ).to be nil
|
420
|
+
end
|
421
|
+
end
|
422
|
+
|
423
|
+
describe "timeout on grouped (multi-pattern) failure" do
|
424
|
+
let(:config) {
|
425
|
+
{
|
426
|
+
'match' => {
|
427
|
+
"message" => [
|
428
|
+
"(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
|
429
|
+
"(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
|
430
|
+
"(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
|
431
|
+
"(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
|
432
|
+
"(.*a){20}"
|
433
|
+
]
|
434
|
+
},
|
435
|
+
'timeout_millis' => 750,
|
436
|
+
'timeout_scope' => 'event'
|
437
|
+
}
|
438
|
+
}
|
439
|
+
|
440
|
+
sample( 'b' * 15 + 'c' * 15 + 'd' * 15 + 'e' * 15 + ' ' + 'a' * 20 ) do
|
441
|
+
expect( event.get("tags") ).to include("_groktimeout")
|
442
|
+
expect( event.get("tags") ).not_to include("_grokparsefailure")
|
425
443
|
end
|
426
444
|
end
|
427
445
|
|
428
446
|
describe "tagging on failure" do
|
429
|
-
config
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
tag_on_failure => not_a_match
|
434
|
-
}
|
447
|
+
let(:config) {
|
448
|
+
{
|
449
|
+
'match' => { "message" => "matchme %{NUMBER:fancy}" },
|
450
|
+
'tag_on_failure' => 'not_a_match'
|
435
451
|
}
|
436
|
-
|
452
|
+
}
|
437
453
|
|
438
454
|
sample "matchme 1234" do
|
439
|
-
|
455
|
+
expect( event.get("tags") ).to be nil
|
440
456
|
end
|
441
457
|
|
442
458
|
sample "this will not be matched" do
|
443
|
-
|
459
|
+
expect( event.get("tags") ).to include("not_a_match")
|
444
460
|
end
|
445
461
|
end
|
446
462
|
|
447
463
|
describe "captures named fields even if the whole text matches" do
|
448
|
-
config
|
449
|
-
|
450
|
-
|
451
|
-
match => { "message" => "%{DATE_EU:stimestamp}" }
|
452
|
-
}
|
464
|
+
let(:config) {
|
465
|
+
{
|
466
|
+
'match' => { "message" => "%{DATE_EU:stimestamp}" }
|
453
467
|
}
|
454
|
-
|
468
|
+
}
|
455
469
|
|
456
470
|
sample "11/01/01" do
|
457
|
-
|
471
|
+
expect( event.get("stimestamp") ).to eql "11/01/01"
|
458
472
|
end
|
459
473
|
end
|
460
474
|
|
461
475
|
describe "allow dashes in capture names" do
|
462
|
-
config
|
463
|
-
|
464
|
-
|
465
|
-
match => { "message" => "%{WORD:foo-bar}" }
|
466
|
-
}
|
476
|
+
let(:config) {
|
477
|
+
{
|
478
|
+
'match' => { "message" => "%{WORD:foo-bar}" }
|
467
479
|
}
|
468
|
-
|
480
|
+
}
|
469
481
|
|
470
482
|
sample "hello world" do
|
471
|
-
|
472
|
-
end
|
473
|
-
end
|
474
|
-
|
475
|
-
describe "performance test", :performance => true do
|
476
|
-
event_count = 100000
|
477
|
-
min_rate = 2000
|
478
|
-
|
479
|
-
max_duration = event_count / min_rate
|
480
|
-
input = "Nov 24 01:29:01 -0800"
|
481
|
-
config <<-CONFIG
|
482
|
-
input {
|
483
|
-
generator {
|
484
|
-
count => #{event_count}
|
485
|
-
message => "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
|
486
|
-
}
|
487
|
-
}
|
488
|
-
filter {
|
489
|
-
grok {
|
490
|
-
match => { "message" => "%{SYSLOGLINE}" }
|
491
|
-
overwrite => [ "message" ]
|
492
|
-
}
|
493
|
-
}
|
494
|
-
output { null { } }
|
495
|
-
CONFIG
|
496
|
-
|
497
|
-
2.times do
|
498
|
-
start = Time.now
|
499
|
-
agent do
|
500
|
-
duration = (Time.now - start)
|
501
|
-
puts "filters/grok parse rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s"
|
502
|
-
insist { duration } < max_duration
|
503
|
-
end
|
483
|
+
expect( event.get("foo-bar") ).to eql "hello"
|
504
484
|
end
|
505
485
|
end
|
506
486
|
|
507
487
|
describe "single value match with duplicate-named fields in pattern" do
|
508
|
-
config
|
509
|
-
|
510
|
-
|
511
|
-
match => { "message" => "%{INT:foo}|%{WORD:foo}" }
|
512
|
-
}
|
488
|
+
let(:config) {
|
489
|
+
{
|
490
|
+
'match' => { "message" => "%{INT:foo}|%{WORD:foo}" }
|
513
491
|
}
|
514
|
-
|
492
|
+
}
|
515
493
|
|
516
494
|
sample "hello world" do
|
517
|
-
|
495
|
+
expect( event.get("foo") ).to be_a(String)
|
518
496
|
end
|
519
497
|
|
520
498
|
sample "123 world" do
|
521
|
-
|
499
|
+
expect( event.get("foo") ).to be_a(String)
|
522
500
|
end
|
523
501
|
end
|
524
502
|
|
525
|
-
describe "break_on_match default should be true and first match should exit filter" do
|
526
|
-
config <<-CONFIG
|
527
|
-
filter {
|
528
|
-
grok {
|
529
|
-
match => { "message" => "%{INT:foo}"
|
530
|
-
"somefield" => "%{INT:bar}"}
|
531
|
-
}
|
532
|
-
}
|
533
|
-
CONFIG
|
534
|
-
|
535
|
-
sample("message" => "hello world 123", "somefield" => "testme abc 999") do
|
536
|
-
insist { subject.get("foo") } == "123"
|
537
|
-
insist { subject.get("bar") }.nil?
|
538
|
-
end
|
539
|
-
end
|
540
503
|
|
541
|
-
describe "break_on_match
|
542
|
-
config
|
543
|
-
|
544
|
-
|
545
|
-
match => { "message" => "%{INT:foo}"
|
546
|
-
"somefield" => "%{INT:bar}"}
|
547
|
-
break_on_match => false
|
548
|
-
}
|
504
|
+
describe "break_on_match default should be true" do
|
505
|
+
let(:config) {
|
506
|
+
{
|
507
|
+
'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" }
|
549
508
|
}
|
550
|
-
|
509
|
+
}
|
510
|
+
let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
|
551
511
|
|
552
|
-
|
553
|
-
|
554
|
-
|
512
|
+
it 'exits filter after first match' do
|
513
|
+
expect( event.get("foo") ).to eql '123'
|
514
|
+
expect( event.get("bar") ).to be nil
|
555
515
|
end
|
556
516
|
end
|
557
517
|
|
558
|
-
describe "
|
559
|
-
config
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
break_on_match => false
|
564
|
-
}
|
518
|
+
describe "break_on_match when set to false" do
|
519
|
+
let(:config) {
|
520
|
+
{
|
521
|
+
'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" },
|
522
|
+
'break_on_match' => false
|
565
523
|
}
|
566
|
-
|
567
|
-
|
568
|
-
sample "treebranch" do
|
569
|
-
insist { subject.get("name2") } == "branch"
|
570
|
-
end
|
571
|
-
|
572
|
-
sample "bushbeard" do
|
573
|
-
insist { subject.get("name1") } == "bush"
|
574
|
-
end
|
524
|
+
}
|
525
|
+
let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
|
575
526
|
|
576
|
-
|
577
|
-
|
578
|
-
|
527
|
+
it 'should try all patterns' do
|
528
|
+
expect( event.get("foo") ).to eql '123'
|
529
|
+
expect( event.get("bar") ).to eql '999'
|
579
530
|
end
|
580
531
|
end
|
581
532
|
|
582
|
-
|
583
|
-
config
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
}
|
533
|
+
context "break_on_match default for array input with single grok pattern" do
|
534
|
+
let(:config) {
|
535
|
+
{
|
536
|
+
'match' => { "message" => "%{INT:foo}" },
|
537
|
+
'break_on_match' => false
|
588
538
|
}
|
589
|
-
|
539
|
+
}
|
590
540
|
|
591
|
-
|
592
|
-
|
593
|
-
|
594
|
-
|
541
|
+
describe 'fully matching input' do
|
542
|
+
let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
|
543
|
+
it 'matches' do
|
544
|
+
expect( event.get("foo") ).to eql ["123", "23"]
|
545
|
+
expect( event.get("tags") ).to be nil
|
546
|
+
end
|
595
547
|
end
|
596
548
|
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
549
|
+
describe 'partially matching input' do
|
550
|
+
let(:data) { Hash("message" => ["hello world 123", "abc"]) } # array input, one of them matches
|
551
|
+
it 'matches' do
|
552
|
+
expect( event.get("foo") ).to eql "123"
|
553
|
+
expect( event.get("tags") ).to be nil
|
554
|
+
end
|
601
555
|
end
|
602
556
|
end
|
603
557
|
|
604
558
|
describe "break_on_match = true (default) for array input with multiple grok pattern" do
|
605
|
-
config
|
606
|
-
|
607
|
-
|
608
|
-
match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
|
609
|
-
}
|
559
|
+
let(:config) {
|
560
|
+
{
|
561
|
+
'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
|
610
562
|
}
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
563
|
+
}
|
564
|
+
|
565
|
+
describe 'matching input' do
|
566
|
+
let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
|
567
|
+
it 'matches' do
|
568
|
+
expect( event.get("foo") ).to eql ["123", "23"]
|
569
|
+
expect( event.get("bar") ).to be nil
|
570
|
+
expect( event.get("tags") ).to be nil
|
571
|
+
end
|
618
572
|
end
|
619
573
|
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
574
|
+
describe 'partially matching input' do
|
575
|
+
let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
|
576
|
+
it 'matches' do
|
577
|
+
expect( event.get("bar") ).to eql 'hello'
|
578
|
+
expect( event.get("foo") ).to eql "23"
|
579
|
+
expect( event.get("tags") ).to be nil
|
580
|
+
end
|
625
581
|
end
|
626
582
|
end
|
627
583
|
|
628
584
|
describe "break_on_match = false for array input with multiple grok pattern" do
|
629
|
-
config
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
break_on_match => false
|
634
|
-
}
|
585
|
+
let(:config) {
|
586
|
+
{
|
587
|
+
'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] },
|
588
|
+
'break_on_match' => false
|
635
589
|
}
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
590
|
+
}
|
591
|
+
|
592
|
+
describe 'fully matching input' do
|
593
|
+
let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
|
594
|
+
it 'matches' do
|
595
|
+
expect( event.get("foo") ).to eql ["123", "23"]
|
596
|
+
expect( event.get("bar") ).to eql ["hello", "line"]
|
597
|
+
expect( event.get("tags") ).to be nil
|
598
|
+
end
|
643
599
|
end
|
644
600
|
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
601
|
+
describe 'partially matching input' do
|
602
|
+
let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
|
603
|
+
it 'matches' do
|
604
|
+
expect( event.get("bar") ).to eql ["hello", "line"]
|
605
|
+
expect( event.get("foo") ).to eql "23"
|
606
|
+
expect( event.get("tags") ).to be nil
|
607
|
+
end
|
650
608
|
end
|
651
609
|
end
|
652
610
|
|
653
611
|
describe "grok with unicode" do
|
654
|
-
config
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
match => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
|
659
|
-
}
|
612
|
+
let(:config) {
|
613
|
+
{
|
614
|
+
#'match' => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
|
615
|
+
'match' => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
|
660
616
|
}
|
661
|
-
|
617
|
+
}
|
662
618
|
|
663
619
|
sample "<22>Jan 4 07:50:46 mailmaster postfix/policy-spf[9454]: : SPF permerror (Junk encountered in record 'v=spf1 mx a:mail.domain.no ip4:192.168.0.4 �all'): Envelope-from: email@domain.no" do
|
664
|
-
|
665
|
-
|
666
|
-
|
620
|
+
expect( event.get("tags") ).to be nil
|
621
|
+
expect( event.get("syslog_pri") ).to eql "22"
|
622
|
+
expect( event.get("syslog_program") ).to eql "postfix/policy-spf"
|
667
623
|
end
|
668
624
|
end
|
669
625
|
|
670
|
-
describe
|
671
|
-
|
672
|
-
|
673
|
-
|
626
|
+
describe "grok with nil coerced value" do
|
627
|
+
let(:config) {
|
628
|
+
{
|
629
|
+
'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
|
630
|
+
}
|
631
|
+
}
|
674
632
|
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
@file.write('WORD \b[2-5]\b')
|
679
|
-
@file.close
|
633
|
+
sample "test 28.4ms" do
|
634
|
+
expect( event.get("duration") ).to eql 28.4
|
635
|
+
expect( event.get("tags") ).to be nil
|
680
636
|
end
|
681
637
|
|
682
|
-
|
683
|
-
|
638
|
+
sample "test N/A" do
|
639
|
+
expect( event.to_hash ).not_to include("duration")
|
640
|
+
expect( event.get("tags") ).to be nil
|
684
641
|
end
|
685
642
|
|
686
|
-
sample
|
687
|
-
|
643
|
+
sample "test abc" do
|
644
|
+
expect( event.get("duration") ).to be nil
|
645
|
+
expect( event.get("tags") ).to eql ["_grokparsefailure"]
|
688
646
|
end
|
647
|
+
end
|
689
648
|
|
690
|
-
|
691
|
-
|
692
|
-
|
649
|
+
describe "grok with nil coerced value and keep_empty_captures" do
|
650
|
+
let(:config) {
|
651
|
+
{
|
652
|
+
'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" },
|
653
|
+
'keep_empty_captures' => true
|
654
|
+
}
|
655
|
+
}
|
656
|
+
|
657
|
+
sample "test N/A" do
|
658
|
+
expect( event.to_hash ).to include("duration")
|
659
|
+
expect( event.get("tags") ).to be nil
|
693
660
|
end
|
694
661
|
end
|
695
662
|
|
696
|
-
describe
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
663
|
+
describe "grok with no coercion" do
|
664
|
+
let(:config) {
|
665
|
+
{
|
666
|
+
'match' => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" },
|
667
|
+
}
|
668
|
+
}
|
701
669
|
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
@file1.write('WORD \b[2-5]\b')
|
706
|
-
@file1.close
|
707
|
-
@file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
|
708
|
-
@file2.write('WORD \b[0-1]\b')
|
709
|
-
@file2.close
|
670
|
+
sample "test 28.4ms" do
|
671
|
+
expect( event.get("duration") ).to eql '28.4'
|
672
|
+
expect( event.get("tags") ).to be nil
|
710
673
|
end
|
711
674
|
|
712
|
-
|
713
|
-
|
675
|
+
sample "test N/A" do
|
676
|
+
expect( event.get("duration") ).to be nil
|
677
|
+
expect( event.get("tags") ).to be nil
|
714
678
|
end
|
679
|
+
end
|
715
680
|
|
716
|
-
|
717
|
-
|
718
|
-
|
681
|
+
describe "opening/closing" do
|
682
|
+
let(:config) { { "match" => {"message" => "A"} } }
|
683
|
+
let(:message) { 'AAA' }
|
719
684
|
|
720
|
-
|
721
|
-
|
722
|
-
File.unlink @file2
|
723
|
-
FileUtils.remove_entry tmpdir
|
724
|
-
FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
|
685
|
+
it "should close cleanly" do
|
686
|
+
expect { subject.do_close }.not_to raise_error
|
725
687
|
end
|
726
688
|
end
|
727
689
|
|
728
|
-
describe "
|
729
|
-
|
730
|
-
|
690
|
+
describe "after grok when the event is JSON serialised the field values are unchanged" do
|
691
|
+
let(:config) {
|
692
|
+
{
|
693
|
+
'match' => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"],
|
694
|
+
'remove_field' => ["message","severity"],
|
695
|
+
'add_tag' => ["ssh_failure"]
|
696
|
+
}
|
697
|
+
}
|
731
698
|
|
732
|
-
|
733
|
-
|
734
|
-
|
735
|
-
|
736
|
-
|
737
|
-
@file4.write('WORD \b[2-5]\b')
|
738
|
-
@file4.close
|
739
|
-
end
|
699
|
+
sample('{"facility":"auth","message":"Failed password for testuser from 1.1.1.1 port 22"}') do
|
700
|
+
expect( event.get("username") ).to eql "testuser"
|
701
|
+
expect( event.get("port") ).to eql "22"
|
702
|
+
expect( event.get("src_ip") ).to eql "1.1.1.1"
|
703
|
+
expect( LogStash::Json.dump(event.get('username')) ).to eql "\"testuser\""
|
740
704
|
|
741
|
-
|
742
|
-
|
705
|
+
expect( event.to_json ).to match %r|"src_ip":"1.1.1.1"|
|
706
|
+
expect( event.to_json ).to match %r|"@timestamp":"#{Regexp.escape(event.get('@timestamp').to_s)}"|
|
707
|
+
expect( event.to_json ).to match %r|"port":"22"|
|
708
|
+
expect( event.to_json ).to match %r|"@version":"1"|
|
709
|
+
expect( event.to_json ).to match %r|"username"|i
|
710
|
+
expect( event.to_json ).to match %r|"testuser"|
|
711
|
+
expect( event.to_json ).to match %r|"tags":\["ssh_failure"\]|
|
743
712
|
end
|
713
|
+
end
|
744
714
|
|
745
|
-
|
746
|
-
|
747
|
-
|
715
|
+
describe "grok with inline pattern definition successfully extracts fields" do
|
716
|
+
let(:config) {
|
717
|
+
{
|
718
|
+
'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" },
|
719
|
+
'pattern_definitions' => {
|
720
|
+
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
|
721
|
+
"MY_PATTERN" => "%{YEAR}"
|
722
|
+
}
|
723
|
+
}
|
724
|
+
}
|
748
725
|
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
|
726
|
+
sample "Mon Dec 26 16:22:08 2016 error 2020" do
|
727
|
+
expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
|
728
|
+
expect( event.get("level") ).to eql "error"
|
729
|
+
expect( event.get("hindsight") ).to eql "2020"
|
753
730
|
end
|
754
731
|
end
|
755
732
|
|
756
|
-
describe
|
757
|
-
|
758
|
-
|
733
|
+
describe "grok with inline pattern definition overwrites existing pattern definition" do
|
734
|
+
let(:config) {
|
735
|
+
{
|
736
|
+
'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" },
|
737
|
+
# loglevel was previously ([Aa]lert|ALERT|[Tt]...
|
738
|
+
'pattern_definitions' => {
|
739
|
+
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
|
740
|
+
"LOGLEVEL" => "%{NUMBER}"
|
741
|
+
}
|
742
|
+
}
|
743
|
+
}
|
759
744
|
|
760
|
-
|
761
|
-
|
762
|
-
|
763
|
-
@file3.close
|
764
|
-
Dir.mkdir(File.join(tmpdir, "subdir"))
|
745
|
+
sample "Mon Dec 26 16:22:08 2016 9999" do
|
746
|
+
expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
|
747
|
+
expect( event.get("level") ).to eql "9999"
|
765
748
|
end
|
749
|
+
end
|
766
750
|
|
751
|
+
context 'when timeouts are explicitly disabled' do
|
767
752
|
let(:config) do
|
768
|
-
|
753
|
+
{
|
754
|
+
"timeout_millis" => 0
|
755
|
+
}
|
769
756
|
end
|
770
757
|
|
771
|
-
|
772
|
-
|
773
|
-
|
758
|
+
context 'when given a pathological input', slow: true do
|
759
|
+
let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
|
760
|
+
let(:config) { super().merge("match" => { "message" => "(.*a){30}" }) }
|
774
761
|
|
775
|
-
|
776
|
-
|
777
|
-
|
762
|
+
it 'blocks for at least 3 seconds' do
|
763
|
+
blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
|
764
|
+
expect do
|
765
|
+
Timeout.timeout(3, blocking_exception_class) do
|
766
|
+
subject.filter(event)
|
767
|
+
end
|
768
|
+
end.to raise_exception(blocking_exception_class)
|
769
|
+
end
|
778
770
|
end
|
779
771
|
end
|
772
|
+
end
|
780
773
|
|
781
|
-
|
782
|
-
config <<-CONFIG
|
783
|
-
filter {
|
784
|
-
grok {
|
785
|
-
match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
|
786
|
-
}
|
787
|
-
}
|
788
|
-
CONFIG
|
774
|
+
describe LogStash::Filters::Grok do
|
789
775
|
|
790
|
-
|
791
|
-
|
792
|
-
insist { subject.get("tags") }.nil?
|
793
|
-
end
|
776
|
+
subject(:grok_filter) { described_class.new(config) }
|
777
|
+
let(:config) { {} }
|
794
778
|
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
779
|
+
context 'when initialized with `ecs_compatibility => v8`' do
|
780
|
+
let(:config) { super().merge("ecs_compatibility" => "v8", "match" => ["message", "%{SYSLOGLINE}"]) }
|
781
|
+
context '#register' do
|
782
|
+
let(:logger_stub) { double('Logger').as_null_object }
|
783
|
+
before(:each) { allow_any_instance_of(described_class).to receive(:logger).and_return(logger_stub)}
|
799
784
|
|
800
|
-
|
801
|
-
|
802
|
-
|
785
|
+
it 'logs a helpful warning about the unreleased v8' do
|
786
|
+
grok_filter.register
|
787
|
+
|
788
|
+
expect(logger_stub).to have_received(:warn).with(a_string_including "preview of the unreleased ECS v8")
|
789
|
+
end
|
803
790
|
end
|
804
791
|
end
|
792
|
+
end
|
805
793
|
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
grok {
|
810
|
-
match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
|
811
|
-
keep_empty_captures => true
|
812
|
-
}
|
813
|
-
}
|
814
|
-
CONFIG
|
794
|
+
describe LogStash::Filters::Grok do
|
795
|
+
describe "(LEGACY)" do
|
796
|
+
describe "patterns in the 'patterns/' dir override core patterns" do
|
815
797
|
|
816
|
-
|
817
|
-
|
818
|
-
insist { subject.get("tags") }.nil?
|
819
|
-
end
|
798
|
+
let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
|
799
|
+
let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
|
820
800
|
|
821
|
-
|
801
|
+
before do
|
802
|
+
FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
|
803
|
+
@file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
|
804
|
+
@file.write('WORD \b[2-5]\b')
|
805
|
+
@file.close
|
806
|
+
end
|
822
807
|
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
grok {
|
827
|
-
match => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" }
|
828
|
-
}
|
829
|
-
}
|
830
|
-
CONFIG
|
808
|
+
let(:config) do
|
809
|
+
'filter { grok { match => { "message" => "%{WORD:word}" } } }'
|
810
|
+
end
|
831
811
|
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
end
|
812
|
+
sample("message" => 'hello') do
|
813
|
+
expect(subject.get("tags")).to eql ["_grokparsefailure"]
|
814
|
+
end
|
836
815
|
|
837
|
-
|
838
|
-
|
839
|
-
|
816
|
+
after do
|
817
|
+
File.unlink @file
|
818
|
+
FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
|
819
|
+
end
|
840
820
|
end
|
841
|
-
end
|
842
821
|
|
843
|
-
|
844
|
-
let(:config) { {"match" => {"message" => "A"}} }
|
845
|
-
subject(:plugin) do
|
846
|
-
::LogStash::Filters::Grok.new(config)
|
847
|
-
end
|
822
|
+
describe "patterns in custom dir override those in 'patterns/' dir" do
|
848
823
|
|
849
|
-
|
850
|
-
|
851
|
-
|
824
|
+
let(:tmpdir) { Stud::Temporary.directory }
|
825
|
+
let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
|
826
|
+
let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
|
852
827
|
|
853
|
-
|
854
|
-
|
855
|
-
|
856
|
-
|
828
|
+
before do
|
829
|
+
FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
|
830
|
+
@file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
|
831
|
+
@file1.write('WORD \b[2-5]\b')
|
832
|
+
@file1.close
|
833
|
+
@file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
|
834
|
+
@file2.write('WORD \b[0-1]\b')
|
835
|
+
@file2.close
|
836
|
+
end
|
857
837
|
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
CONFIG
|
838
|
+
let(:config) do
|
839
|
+
"filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
|
840
|
+
end
|
862
841
|
|
863
|
-
|
864
|
-
|
865
|
-
|
866
|
-
insist { subject.get("src_ip") } == "1.1.1.1"
|
867
|
-
insist { LogStash::Json.dump(subject.get('username')) } == "\"testuser\""
|
842
|
+
sample("message" => '0') do
|
843
|
+
expect(subject.get("tags")).to be nil
|
844
|
+
end
|
868
845
|
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]|
|
846
|
+
after do
|
847
|
+
File.unlink @file1
|
848
|
+
File.unlink @file2
|
849
|
+
FileUtils.remove_entry tmpdir
|
850
|
+
FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
|
851
|
+
end
|
876
852
|
end
|
877
|
-
end
|
878
853
|
|
879
|
-
|
880
|
-
config <<-CONFIG
|
881
|
-
filter {
|
882
|
-
grok {
|
883
|
-
match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" }
|
884
|
-
pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
|
885
|
-
"MY_PATTERN" => "%{YEAR}"}
|
886
|
-
}
|
887
|
-
}
|
888
|
-
CONFIG
|
854
|
+
describe "patterns with file glob" do
|
889
855
|
|
890
|
-
|
891
|
-
insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
|
892
|
-
insist { subject.get("level") } == "error"
|
893
|
-
insist { subject.get("hindsight") } == "2020"
|
894
|
-
end
|
895
|
-
end
|
856
|
+
let(:tmpdir) { Stud::Temporary.directory }
|
896
857
|
|
897
|
-
|
898
|
-
|
899
|
-
|
900
|
-
|
901
|
-
|
902
|
-
|
903
|
-
|
904
|
-
|
905
|
-
}
|
906
|
-
}
|
907
|
-
CONFIG
|
858
|
+
before do
|
859
|
+
@file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
|
860
|
+
@file3.write('WORD \b[0-1]\b')
|
861
|
+
@file3.close
|
862
|
+
@file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
|
863
|
+
@file4.write('WORD \b[2-5]\b')
|
864
|
+
@file4.close
|
865
|
+
end
|
908
866
|
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
end
|
913
|
-
end
|
867
|
+
let(:config) do
|
868
|
+
"filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
|
869
|
+
end
|
914
870
|
|
871
|
+
sample("message" => '0') do
|
872
|
+
expect(subject.get("tags")).to be nil
|
873
|
+
end
|
915
874
|
|
916
|
-
|
917
|
-
|
918
|
-
|
919
|
-
|
920
|
-
|
875
|
+
after do
|
876
|
+
File.unlink @file3
|
877
|
+
File.unlink @file4
|
878
|
+
FileUtils.remove_entry tmpdir
|
879
|
+
end
|
921
880
|
end
|
922
881
|
|
923
|
-
|
924
|
-
let(:event) { LogStash::Event.new(data) }
|
882
|
+
describe "patterns with file glob on directory that contains subdirectories" do
|
925
883
|
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
|
884
|
+
let(:tmpdir) { Stud::Temporary.directory }
|
885
|
+
|
886
|
+
before do
|
887
|
+
@file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
|
888
|
+
@file3.write('WORD \b[0-1]\b')
|
889
|
+
@file3.close
|
890
|
+
Dir.mkdir(File.join(tmpdir, "subdir"))
|
891
|
+
end
|
892
|
+
|
893
|
+
let(:config) do
|
894
|
+
"filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
|
895
|
+
end
|
896
|
+
|
897
|
+
sample("message" => '0') do
|
898
|
+
expect(subject.get("tags")).to be nil
|
899
|
+
end
|
900
|
+
|
901
|
+
after do
|
902
|
+
File.unlink @file3
|
903
|
+
FileUtils.remove_entry tmpdir
|
904
|
+
end
|
905
|
+
end
|
906
|
+
|
907
|
+
describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
|
908
|
+
config <<-CONFIG
|
909
|
+
filter {
|
910
|
+
grok {
|
911
|
+
match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
|
912
|
+
break_on_match => false
|
930
913
|
}
|
914
|
+
}
|
915
|
+
CONFIG
|
916
|
+
|
917
|
+
sample "treebranch" do
|
918
|
+
expect(subject.get("name2")).to eql "branch"
|
931
919
|
end
|
932
920
|
|
933
|
-
|
934
|
-
|
935
|
-
|
921
|
+
sample "bushbeard" do
|
922
|
+
expect(subject.get("name1")).to eql "bush"
|
923
|
+
end
|
936
924
|
|
937
|
-
|
938
|
-
|
939
|
-
|
940
|
-
Timeout.timeout(3, blocking_exception_class) do
|
941
|
-
subject.filter(event)
|
942
|
-
end
|
943
|
-
end.to raise_exception(blocking_exception_class)
|
944
|
-
end
|
925
|
+
sample "treebeard" do
|
926
|
+
expect(subject.get("name1")).to eql "tree"
|
927
|
+
expect(subject.get("name2")).to eql "beard"
|
945
928
|
end
|
946
929
|
end
|
947
930
|
end
|
948
|
-
end
|
931
|
+
end
|