logstash-patterns-core 4.1.2 → 4.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +119 -0
- data/Gemfile +8 -1
- data/LICENSE +199 -10
- data/README.md +12 -19
- data/lib/logstash/patterns/core.rb +11 -3
- data/logstash-patterns-core.gemspec +1 -1
- data/patterns/ecs-v1/aws +28 -0
- data/patterns/ecs-v1/bacula +53 -0
- data/patterns/ecs-v1/bind +13 -0
- data/patterns/ecs-v1/bro +30 -0
- data/patterns/ecs-v1/exim +26 -0
- data/patterns/ecs-v1/firewalls +111 -0
- data/patterns/ecs-v1/grok-patterns +95 -0
- data/patterns/ecs-v1/haproxy +40 -0
- data/patterns/ecs-v1/httpd +17 -0
- data/patterns/ecs-v1/java +34 -0
- data/patterns/ecs-v1/junos +13 -0
- data/patterns/ecs-v1/linux-syslog +16 -0
- data/patterns/{maven → ecs-v1/maven} +0 -0
- data/patterns/ecs-v1/mcollective +4 -0
- data/patterns/ecs-v1/mongodb +7 -0
- data/patterns/ecs-v1/nagios +124 -0
- data/patterns/ecs-v1/postgresql +2 -0
- data/patterns/ecs-v1/rails +13 -0
- data/patterns/ecs-v1/redis +3 -0
- data/patterns/ecs-v1/ruby +2 -0
- data/patterns/ecs-v1/squid +6 -0
- data/patterns/ecs-v1/zeek +33 -0
- data/patterns/{aws → legacy/aws} +1 -1
- data/patterns/{bacula → legacy/bacula} +5 -5
- data/patterns/legacy/bind +3 -0
- data/patterns/{bro → legacy/bro} +0 -0
- data/patterns/{exim → legacy/exim} +8 -2
- data/patterns/{firewalls → legacy/firewalls} +2 -2
- data/patterns/{grok-patterns → legacy/grok-patterns} +4 -4
- data/patterns/{haproxy → legacy/haproxy} +1 -1
- data/patterns/{httpd → legacy/httpd} +2 -2
- data/patterns/{java → legacy/java} +1 -3
- data/patterns/{junos → legacy/junos} +0 -0
- data/patterns/{linux-syslog → legacy/linux-syslog} +0 -0
- data/patterns/legacy/maven +1 -0
- data/patterns/{mcollective → legacy/mcollective} +0 -0
- data/patterns/{mcollective-patterns → legacy/mcollective-patterns} +0 -0
- data/patterns/{mongodb → legacy/mongodb} +0 -0
- data/patterns/{nagios → legacy/nagios} +1 -1
- data/patterns/{postgresql → legacy/postgresql} +0 -0
- data/patterns/{rails → legacy/rails} +0 -0
- data/patterns/{redis → legacy/redis} +0 -0
- data/patterns/{ruby → legacy/ruby} +0 -0
- data/patterns/legacy/squid +4 -0
- data/spec/patterns/aws_spec.rb +395 -0
- data/spec/patterns/bacula_spec.rb +367 -0
- data/spec/patterns/bind_spec.rb +92 -0
- data/spec/patterns/bro_spec.rb +613 -0
- data/spec/patterns/core_spec.rb +260 -15
- data/spec/patterns/exim_spec.rb +201 -0
- data/spec/patterns/firewalls_spec.rb +707 -66
- data/spec/patterns/haproxy_spec.rb +253 -28
- data/spec/patterns/httpd_spec.rb +248 -86
- data/spec/patterns/java_spec.rb +375 -0
- data/spec/patterns/junos_spec.rb +101 -0
- data/spec/patterns/mcollective_spec.rb +35 -0
- data/spec/patterns/mongodb_spec.rb +170 -33
- data/spec/patterns/nagios_spec.rb +299 -78
- data/spec/patterns/netscreen_spec.rb +123 -0
- data/spec/patterns/rails3_spec.rb +87 -29
- data/spec/patterns/redis_spec.rb +216 -140
- data/spec/patterns/shorewall_spec.rb +85 -74
- data/spec/patterns/squid_spec.rb +139 -0
- data/spec/patterns/syslog_spec.rb +266 -22
- data/spec/spec_helper.rb +83 -5
- metadata +70 -31
- data/patterns/bind +0 -3
- data/patterns/squid +0 -4
- data/spec/patterns/bro.rb +0 -126
- data/spec/patterns/s3_spec.rb +0 -173
@@ -0,0 +1,613 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "spec_helper"
|
3
|
+
require "logstash/patterns/core"
|
4
|
+
|
5
|
+
describe_pattern "BRO_HTTP", ['legacy', 'ecs-v1'] do
|
6
|
+
|
7
|
+
let(:message) do # old BRO logging format
|
8
|
+
"1432555199.633017 COpk6E3vkURP8QQNKl 192.168.9.35 55281 178.236.7.146 80 4 POST www.amazon.it /xa/dealcontent/v2/GetDeals?nocache=1432555199326 http://www.amazon.it/ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 223 1859 200 OK - - - (empty) kares - - FrLEcY3AUPKdcYGf29 text/plain FOJpbGzIMh9syPxH8 text/plain"
|
9
|
+
end
|
10
|
+
|
11
|
+
it "matches a simple message" do
|
12
|
+
expect(pattern).to match(message)
|
13
|
+
end
|
14
|
+
|
15
|
+
it "generates the ts field" do
|
16
|
+
if ecs_compatibility?
|
17
|
+
expect(grok).to include("timestamp" => "1432555199.633017")
|
18
|
+
else
|
19
|
+
expect(grok).to include("ts" => "1432555199.633017")
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
it "generates the uid field" do
|
24
|
+
if ecs_compatibility?
|
25
|
+
expect(grok).to include("zeek" => hash_including("session_id" => "COpk6E3vkURP8QQNKl"))
|
26
|
+
else
|
27
|
+
expect(grok).to include("uid" => "COpk6E3vkURP8QQNKl")
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
it "generates the orig_ fields" do
|
32
|
+
if ecs_compatibility?
|
33
|
+
expect(grok).to include("source" => { "ip" => "192.168.9.35", "port" => 55281 })
|
34
|
+
else
|
35
|
+
expect(grok).to include("orig_h" => "192.168.9.35", "orig_p" => "55281")
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
it "generates the resp_ fields" do
|
40
|
+
if ecs_compatibility?
|
41
|
+
expect(grok).to include("destination" => { "ip" => "178.236.7.146", "port" => 80 })
|
42
|
+
|
43
|
+
else
|
44
|
+
expect(grok).to include("resp_h" => "178.236.7.146", "resp_p" => "80")
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
it "generates the trans_depth field" do
|
49
|
+
if ecs_compatibility?
|
50
|
+
expect(grok).to include("zeek" => hash_including("http" => hash_including("trans_depth" => 4)))
|
51
|
+
else
|
52
|
+
expect(grok).to include("trans_depth" => "4")
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
it "generates the method/referrer field" do
|
57
|
+
if ecs_compatibility?
|
58
|
+
expect(grok).to include("http" => hash_including("request" => hash_including("method" => "POST", "referrer" => "http://www.amazon.it/" )))
|
59
|
+
else
|
60
|
+
expect(grok).to include("method" => "POST")
|
61
|
+
expect(grok).to include("referrer" => "http://www.amazon.it/")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
it "generates the domain/uri/referrer & username/password field" do
|
66
|
+
if ecs_compatibility?
|
67
|
+
expect(grok).to include("url" => hash_including("domain" => "www.amazon.it"))
|
68
|
+
expect(grok).to include("url" => hash_including("original" => "/xa/dealcontent/v2/GetDeals?nocache=1432555199326"))
|
69
|
+
|
70
|
+
expect(grok).to include("url" => hash_including("username" => "kares"))
|
71
|
+
expect(grok['url'].keys).to_not include("password")
|
72
|
+
else
|
73
|
+
expect(grok).to include("domain" => "www.amazon.it")
|
74
|
+
expect(grok).to include("uri" => "/xa/dealcontent/v2/GetDeals?nocache=1432555199326")
|
75
|
+
|
76
|
+
expect(grok).to include("username" => "kares")
|
77
|
+
expect(grok).to include("password" => "-")
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
it "generates the user_agent field" do
|
82
|
+
if ecs_compatibility?
|
83
|
+
expect(grok).to include("user_agent" => { "original" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36" })
|
84
|
+
else
|
85
|
+
expect(grok).to include("user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36")
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
it "generates the request_body_len/response_body_len fields" do
|
90
|
+
if ecs_compatibility?
|
91
|
+
expect(grok).to include("http" => hash_including("request" => hash_including("body" => { "bytes" => 223 })))
|
92
|
+
expect(grok).to include("http" => hash_including("response" => hash_including("body" => { "bytes" => 1859 })))
|
93
|
+
else
|
94
|
+
expect(grok).to include("request_body_len" => "223")
|
95
|
+
expect(grok).to include("response_body_len" => "1859")
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
it "generates the status_ fields" do
|
100
|
+
if ecs_compatibility?
|
101
|
+
expect(grok).to include("http" => hash_including("response" => hash_including("status_code" => 200)))
|
102
|
+
expect(grok).to include("zeek" => hash_including("http" => hash_including("status_msg" => "OK")))
|
103
|
+
else
|
104
|
+
expect(grok).to include("status_code" => "200", "status_msg" => "OK")
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
it "generates the info_ fields" do
|
109
|
+
if ecs_compatibility?
|
110
|
+
expect(grok['zeek']['http'].keys).to_not include("info_code", "info_msg")
|
111
|
+
else
|
112
|
+
expect(grok).to include("info_code" => "-", "info_msg" => "-")
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
it "generates the filename field" do
|
117
|
+
if ecs_compatibility?
|
118
|
+
expect(grok['zeek']['http'].keys).to_not include("filename")
|
119
|
+
else
|
120
|
+
expect(grok).to include("filename" => "-")
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
it "generates the bro_tags field" do
|
125
|
+
if ecs_compatibility?
|
126
|
+
expect(grok['zeek']['http'].keys).to_not include("tags")
|
127
|
+
else
|
128
|
+
expect(grok).to include("bro_tags" => "(empty)")
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
it "generates the proxied field" do
|
133
|
+
if ecs_compatibility?
|
134
|
+
expect(grok['zeek']['http'].keys).to_not include("proxied")
|
135
|
+
else
|
136
|
+
expect(grok).to include("proxied" => "-")
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
it "generates the orig_ fields" do
|
141
|
+
if ecs_compatibility?
|
142
|
+
expect(grok).to include("zeek" => hash_including("http" => hash_including("orig_fuids" => "FrLEcY3AUPKdcYGf29")))
|
143
|
+
expect(grok).to include("http" => hash_including("request" => hash_including("mime_type" => "text/plain")))
|
144
|
+
else
|
145
|
+
expect(grok).to include("orig_fuids" => "FrLEcY3AUPKdcYGf29")
|
146
|
+
expect(grok).to include("orig_mime_types" => "text/plain")
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
it "generates the resp_ fields" do
|
151
|
+
if ecs_compatibility?
|
152
|
+
expect(grok).to include("zeek" => hash_including("http" => hash_including("resp_fuids" => "FOJpbGzIMh9syPxH8")))
|
153
|
+
expect(grok).to include("http" => hash_including("response" => hash_including("mime_type" => "text/plain")))
|
154
|
+
else
|
155
|
+
expect(grok).to include("resp_fuids" => "FOJpbGzIMh9syPxH8")
|
156
|
+
expect(grok).to include("resp_mime_types" => "text/plain")
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
context '(zeek) updated log format' do
|
161
|
+
|
162
|
+
let(:message) do # ZEEK
|
163
|
+
'1602164975.587600 Ct73QY3M7T5dikxggf 192.168.122.59 55240 93.184.220.29 80 1 - - - - 1.1 - - 0 471 200 OK - - (empty) - - - - - - FPGXN33wAFL8MPKJXl - application/ocsp-response'
|
164
|
+
end
|
165
|
+
|
166
|
+
it 'matches in legacy mode' do
|
167
|
+
unless ecs_compatibility? # wrong but backwards compatibility
|
168
|
+
expect(grok).to include("domain" => "1.1") # due GREEDYDATA: "method" => "-\t-\t-\t-"
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
it 'no longer matches in ecs mode' do
|
173
|
+
expect(grok['tags']).to include("_grokparsefailure") if ecs_compatibility?
|
174
|
+
end
|
175
|
+
|
176
|
+
end
|
177
|
+
|
178
|
+
end
|
179
|
+
|
180
|
+
describe_pattern "ZEEK_HTTP", ['ecs-v1'] do
|
181
|
+
|
182
|
+
context "long message" do
|
183
|
+
|
184
|
+
let(:message) do
|
185
|
+
"1333458850.375568 ClEkJM2Vm5giqnMf4h 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf 1.1 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 - 0 56320 206 Partial Content - - (empty) - - - - - - FpmJd62pFQcZ3gcUgl - -"
|
186
|
+
end
|
187
|
+
|
188
|
+
it "matches" do
|
189
|
+
expect(grok).to include("timestamp"=>"1333458850.375568")
|
190
|
+
expect(grok).to include("zeek" => hash_including("session_id"=>"ClEkJM2Vm5giqnMf4h"))
|
191
|
+
expect(grok).to include("http" => {
|
192
|
+
"response" => { "body" =>{ "bytes" => 56320 }, "status_code"=>206 },
|
193
|
+
"request" => { "body" => { "bytes" => 0 }, "referrer"=>"http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf", "method"=>"GET"},
|
194
|
+
"version"=>"1.1"
|
195
|
+
})
|
196
|
+
expect(grok).to include("user_agent"=>{"original"=>"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11"})
|
197
|
+
expect(grok).to include("url"=>{
|
198
|
+
"domain"=>"o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com",
|
199
|
+
"original"=>"/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1"
|
200
|
+
})
|
201
|
+
expect(grok).to include("destination"=>{"port"=>80, "ip"=>"79.101.110.141"}, "source"=>{"port"=>1923, "ip"=>"10.131.47.185"})
|
202
|
+
expect(grok).to include("zeek" => hash_including("http"=>{"resp_fuids"=>"FpmJd62pFQcZ3gcUgl", "status_msg"=>"Partial Content", "trans_depth"=>1}))
|
203
|
+
end
|
204
|
+
|
205
|
+
end
|
206
|
+
|
207
|
+
context "sample message" do
|
208
|
+
|
209
|
+
let(:message) do
|
210
|
+
"1602165002.455618 CWGOypTfRypTC5C4g 192.168.122.59 44136 216.58.201.110 80 1 - - - - 1.1 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0 https://example.com 0 219 301 Moved Permanently - - FOO,BAR - - - - - - FeJ7iiVorMXoLlRK - text/html"
|
211
|
+
end
|
212
|
+
|
213
|
+
it "matches" do
|
214
|
+
expect(grok).to include("timestamp"=>"1602165002.455618")
|
215
|
+
expect(grok).to include("user_agent"=>{"original"=>"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0"})
|
216
|
+
expect(grok).to include("http" => {
|
217
|
+
"request" => { "body" => {"bytes" => 0 } },
|
218
|
+
"response" => { "status_code" => 301, "body" =>{ "bytes" => 219 }, "mime_type" => 'text/html' },
|
219
|
+
"version"=>"1.1"
|
220
|
+
})
|
221
|
+
expect(grok).to include("zeek" => {
|
222
|
+
"session_id" => "CWGOypTfRypTC5C4g",
|
223
|
+
"http" => {
|
224
|
+
"trans_depth"=>1,
|
225
|
+
"origin"=>"https://example.com",
|
226
|
+
"tags"=>"FOO,BAR",
|
227
|
+
"status_msg"=>"Moved Permanently",
|
228
|
+
"resp_fuids"=>"FeJ7iiVorMXoLlRK"}
|
229
|
+
})
|
230
|
+
end
|
231
|
+
|
232
|
+
end
|
233
|
+
|
234
|
+
context 'old (bro) message' do
|
235
|
+
|
236
|
+
let(:message) do
|
237
|
+
"1432555199.633017 COpk6E3vkURP8QQNKl 192.168.9.35 55281 178.236.7.146 80 4 POST www.amazon.it /xa/dealcontent/v2/GetDeals?nocache=1432555199326 http://www.amazon.it/ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 223 1859 200 OK - - - (empty) kares - - FrLEcY3AUPKdcYGf29 text/plain FOJpbGzIMh9syPxH8 text/plain"
|
238
|
+
end
|
239
|
+
|
240
|
+
it 'does not match' do
|
241
|
+
expect(grok['tags']).to include("_grokparsefailure") if ecs_compatibility?
|
242
|
+
end
|
243
|
+
|
244
|
+
end
|
245
|
+
|
246
|
+
context 'old empty (bro) message' do
|
247
|
+
|
248
|
+
let(:message) do # theoretically everything except these is optional:
|
249
|
+
"1432555199.633017 COpk6E3vkURP8QQNKl 192.168.9.35 55281 178.236.7.146 80 0 - - - - - - - - - - - - (empty) - - - - - - -"
|
250
|
+
end
|
251
|
+
|
252
|
+
it 'does not match' do
|
253
|
+
expect(grok['tags']).to include("_grokparsefailure") if ecs_compatibility?
|
254
|
+
end
|
255
|
+
|
256
|
+
end
|
257
|
+
|
258
|
+
end
|
259
|
+
|
260
|
+
describe_pattern 'BRO_DNS', ['legacy', 'ecs-v1'] do
|
261
|
+
|
262
|
+
let(:message) do # old BRO logging format
|
263
|
+
"1359565680.761790 UWkUyAuUGXf 192.168.6.10 53209 192.168.129.36 53 udp 41477 paypal.com 1 C_INTERNET 48 DNSKEY 0 NOERROR F F T F 1 - - F"
|
264
|
+
end
|
265
|
+
|
266
|
+
it 'matches' do
|
267
|
+
if ecs_compatibility?
|
268
|
+
expect(grok).to include("timestamp" => "1359565680.761790")
|
269
|
+
expect(grok).to include("zeek" => hash_including("session_id" => "UWkUyAuUGXf"))
|
270
|
+
expect(grok).to include("source" => { "ip"=>"192.168.6.10", "port"=>53209 }, "destination" => { "ip"=>"192.168.129.36", "port"=>53 })
|
271
|
+
expect(grok).to include("network" => { "transport" => "udp" })
|
272
|
+
expect(grok).to include("dns" => {
|
273
|
+
"id" => 41477,
|
274
|
+
"question" => { "name" => "paypal.com", "type" => "DNSKEY" },
|
275
|
+
"response_code" => "NOERROR",
|
276
|
+
})
|
277
|
+
expect(grok['zeek']).to include("dns" => hash_including("qclass" => 1, "qclass_name" => "C_INTERNET"))
|
278
|
+
expect(grok['zeek']).to include("dns" => hash_including("qtype" => 48)) # beats compatibility
|
279
|
+
expect(grok['zeek']).to include("dns" => hash_including("rcode" => 0)) # beats compatibility
|
280
|
+
# TODO :bool type-casting would be nice
|
281
|
+
expect(grok['zeek']).to include("dns" => hash_including("AA"=>"F", "TC"=>"F", "RD"=>"T", "RA"=>"F"))
|
282
|
+
expect(grok['zeek']).to include("dns" => hash_including("Z" => 1)) # beats drops this field
|
283
|
+
expect(grok['zeek']).to include("dns" => hash_including("rejected" => "F"))
|
284
|
+
expect(grok['zeek']['dns'].keys).to_not include 'TTLs', 'answers'
|
285
|
+
else
|
286
|
+
expect(grok).to include(
|
287
|
+
"ts"=>"1359565680.761790", "uid"=>"UWkUyAuUGXf",
|
288
|
+
"orig_h"=>"192.168.6.10", "orig_p"=>"53209",
|
289
|
+
"resp_h" => "192.168.129.36", "resp_p"=>"53",
|
290
|
+
"proto"=>"udp",
|
291
|
+
"trans_id"=>"41477",
|
292
|
+
"query"=>"paypal.com",
|
293
|
+
"qclass"=>"1", "qclass_name"=>"C_INTERNET",
|
294
|
+
"qtype"=>"48", "qtype_name"=>"DNSKEY",
|
295
|
+
"rcode"=>"0", "rcode_name"=>"NOERROR",
|
296
|
+
"AA"=>"F", "TC"=>"F",
|
297
|
+
"RD"=>"T", "RA"=>"F",
|
298
|
+
"Z"=>"1",
|
299
|
+
"answers"=>"-",
|
300
|
+
"TTLs"=>"-",
|
301
|
+
"rejected"=>"F",
|
302
|
+
)
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
context 'optional fields' do
|
307
|
+
|
308
|
+
let(:message) do
|
309
|
+
"1359565680.761790 UWkUyAuUGXf 192.168.6.10 53209 192.168.129.36 53 udp - - - - - - - - F F F F 0 - - -"
|
310
|
+
# AA/TC/RD/RA are optional with a F default, Z is optional with a 0 default
|
311
|
+
end
|
312
|
+
|
313
|
+
it 'matches (only) fields present' do
|
314
|
+
if ecs_compatibility?
|
315
|
+
expect(grok).to include("timestamp" => "1359565680.761790")
|
316
|
+
expect(grok).to include("source" => { "ip"=>"192.168.6.10", "port"=>53209 }, "destination" => { "ip"=>"192.168.129.36", "port"=>53 })
|
317
|
+
expect(grok).to include("network" => { "transport"=>"udp" })
|
318
|
+
expect(grok).to include("zeek" => { "session_id" => "UWkUyAuUGXf", "dns" => { "AA"=>"F", "TC"=>"F", "RD"=>"F", "RA"=>"F", "Z"=>0 } })
|
319
|
+
expect(grok.keys).to_not include('dns')
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
end
|
324
|
+
|
325
|
+
context '(zeek) updated log format' do
|
326
|
+
|
327
|
+
let(:message) do
|
328
|
+
"1359565680.761790 CHhAvVGS1DHFjwGM9 192.168.6.10 53209 192.168.129.36 53 udp 41477 0.075138 paypal.com 1 C_INTERNET 48 DNSKEY 0 NOERROR F F T T 1 DNSKEY 5,DNSKEY 5,RRSIG 48 paypal.com,RRSIG 48 paypal.com 455.000000,455.000000,455.000000,455.000000 F"
|
329
|
+
end
|
330
|
+
|
331
|
+
it 'no longer matches in ecs mode' do
|
332
|
+
expect(grok['tags']).to include("_grokparsefailure") if ecs_compatibility?
|
333
|
+
end
|
334
|
+
|
335
|
+
end
|
336
|
+
|
337
|
+
end
|
338
|
+
|
339
|
+
describe_pattern "ZEEK_DNS", ['ecs-v1'] do
|
340
|
+
|
341
|
+
let(:message) do
|
342
|
+
"1359565680.761790 CHhAvVGS1DHFjwGM9 192.168.6.10 53209 192.168.129.36 53 udp 41477 0.075138 paypal.com 1 C_INTERNET 48 DNSKEY 0 NOERROR F F T T 1 DNSKEY 5,DNSKEY 5,RRSIG 48 paypal.com,RRSIG 48 paypal.com 455.000000,455.000000,455.000000,455.000000 F"
|
343
|
+
end
|
344
|
+
|
345
|
+
it 'matches' do
|
346
|
+
expect(grok).to include("timestamp" => "1359565680.761790")
|
347
|
+
expect(grok).to include("destination"=>{"ip"=>"192.168.129.36", "port"=>53}, "source"=>{"ip"=>"192.168.6.10", "port"=>53209})
|
348
|
+
expect(grok).to include("network" => {"transport"=>"udp"})
|
349
|
+
expect(grok).to include("dns"=>{"question"=>{"type"=>"DNSKEY", "name"=>"paypal.com"}, "response_code"=>"NOERROR", "id"=>41477})
|
350
|
+
expect(grok).to include("zeek"=>{
|
351
|
+
"session_id"=>"CHhAvVGS1DHFjwGM9",
|
352
|
+
"dns"=>{
|
353
|
+
"rtt" => 0.075138 ,
|
354
|
+
"qclass"=>1, "qclass_name"=>"C_INTERNET", "qtype"=>48,
|
355
|
+
"rcode"=>0,
|
356
|
+
"RA"=>"T", "RD"=>"T", "TC"=>"F", "rejected"=>"F", "AA"=>"F", "Z"=>1,
|
357
|
+
"answers"=>"DNSKEY 5,DNSKEY 5,RRSIG 48 paypal.com,RRSIG 48 paypal.com",
|
358
|
+
"TTLs"=>"455.000000,455.000000,455.000000,455.000000"
|
359
|
+
}
|
360
|
+
})
|
361
|
+
end
|
362
|
+
|
363
|
+
end
|
364
|
+
|
365
|
+
describe_pattern 'BRO_CONN', ['legacy', 'ecs-v1'] do
|
366
|
+
|
367
|
+
let(:message) do
|
368
|
+
"1541350796.901974 C54zqz17PXuBv3HkLg 192.168.0.26 54855 54.85.115.89 443 tcp ssl 0.153642 1147 589 SF T 0 ShADadfF 7 1439 8 921 (empty)"
|
369
|
+
end
|
370
|
+
|
371
|
+
it 'matches' do
|
372
|
+
if ecs_compatibility?
|
373
|
+
expect(grok).to include("timestamp"=>"1541350796.901974")
|
374
|
+
expect(grok).to include("zeek" => hash_including("session_id" => "C54zqz17PXuBv3HkLg"))
|
375
|
+
expect(grok).to include("network"=>{"transport"=>"tcp", "protocol"=>"ssl"})
|
376
|
+
expect(grok).to include("source"=>{
|
377
|
+
"ip"=>"192.168.0.26", "port"=>54855,
|
378
|
+
"packets"=>7, "bytes"=>1439
|
379
|
+
})
|
380
|
+
expect(grok).to include("destination"=>{
|
381
|
+
"ip"=>"54.85.115.89", "port"=>443,
|
382
|
+
"packets"=>8, "bytes"=>921
|
383
|
+
})
|
384
|
+
expect(grok).to include("zeek" => hash_including(
|
385
|
+
"connection" => {
|
386
|
+
"duration"=>0.153642,
|
387
|
+
"orig_bytes"=>1147, "resp_bytes"=>589,
|
388
|
+
"state"=>"SF",
|
389
|
+
"local_orig"=>"T", "missed_bytes"=>0,
|
390
|
+
"history"=>"ShADadfF",
|
391
|
+
}
|
392
|
+
))
|
393
|
+
else
|
394
|
+
expect(grok).to include(
|
395
|
+
"ts"=>"1541350796.901974", "uid"=>"C54zqz17PXuBv3HkLg",
|
396
|
+
"orig_h"=>"192.168.0.26", "orig_p"=>"54855",
|
397
|
+
"resp_h" => "54.85.115.89", "resp_p"=>"443",
|
398
|
+
"proto"=>"tcp",
|
399
|
+
"service"=>"ssl",
|
400
|
+
"duration"=>"0.153642",
|
401
|
+
"orig_bytes"=>"1147", "resp_bytes"=>"589",
|
402
|
+
"conn_state"=>"SF",
|
403
|
+
"local_orig"=>"T",
|
404
|
+
"missed_bytes"=>"0",
|
405
|
+
"history"=>"ShADadfF",
|
406
|
+
"orig_pkts"=>"7", "orig_ip_bytes"=>"1439",
|
407
|
+
"resp_pkts"=>"8", "resp_ip_bytes"=>"921",
|
408
|
+
"tunnel_parents"=>"(empty)",
|
409
|
+
)
|
410
|
+
end
|
411
|
+
end
|
412
|
+
|
413
|
+
context 'updated log format' do # up to version 3.3
|
414
|
+
let(:message) do
|
415
|
+
"1300475168.853899 C4J4Th3PJpwUYZZ6gc 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF - - 0 Dd 1 66 1 117 (empty)"
|
416
|
+
end
|
417
|
+
|
418
|
+
it 'matches' do
|
419
|
+
if ecs_compatibility?
|
420
|
+
expect(grok).to include("zeek" => hash_including("session_id" => "C4J4Th3PJpwUYZZ6gc"))
|
421
|
+
expect(grok).to include("network"=>{"transport"=>"udp", "protocol"=>"dns"})
|
422
|
+
expect(grok).to include("source"=>{
|
423
|
+
"ip"=>"141.142.220.118", "port"=>43927,
|
424
|
+
"packets"=>1, "bytes"=>66
|
425
|
+
})
|
426
|
+
expect(grok).to include("destination"=>{
|
427
|
+
"ip"=>"141.142.2.2", "port"=>53,
|
428
|
+
"packets"=>1, "bytes"=>117
|
429
|
+
})
|
430
|
+
expect(grok).to include("zeek" => hash_including(
|
431
|
+
"connection" => {
|
432
|
+
"duration"=>0.000435,
|
433
|
+
"orig_bytes"=>38, "resp_bytes"=>89,
|
434
|
+
"state"=>"SF",
|
435
|
+
"missed_bytes"=>0,
|
436
|
+
"history"=>"Dd",
|
437
|
+
}
|
438
|
+
))
|
439
|
+
else
|
440
|
+
expect(grok).to include("ts"=>"1300475168.853899")
|
441
|
+
expect(grok).to include("conn_state"=>"SF\t-") # matches incorrectly
|
442
|
+
end
|
443
|
+
end
|
444
|
+
end
|
445
|
+
|
446
|
+
end
|
447
|
+
|
448
|
+
describe_pattern 'ZEEK_CONN', ['ecs-v1'] do
|
449
|
+
|
450
|
+
let(:message) do
|
451
|
+
"1602165198.216970 CfVl4yNLJ4U3ptL12 192.168.122.59 47340 143.204.201.102 443 tcp - 115.669583 0 6238 SHR T F 0 ^hCadcCfA 1 52 21 5091 -"
|
452
|
+
end
|
453
|
+
|
454
|
+
it 'matches' do
|
455
|
+
expect(grok).to include("timestamp" => "1602165198.216970")
|
456
|
+
expect(grok).to include("network" => { "transport" => "tcp" })
|
457
|
+
expect(grok).to include("source" => { "ip"=>"192.168.122.59", "port"=>47340, "packets"=>1, "bytes"=>52 })
|
458
|
+
expect(grok).to include("destination" => { "ip"=>"143.204.201.102", "port"=>443, "packets"=>21, "bytes"=>5091 })
|
459
|
+
|
460
|
+
expect(grok).to include("zeek"=>{
|
461
|
+
"connection"=>{
|
462
|
+
"duration"=>115.669583,
|
463
|
+
"orig_bytes"=>0,
|
464
|
+
"resp_bytes"=>6238,
|
465
|
+
"state"=>"SHR",
|
466
|
+
"local_orig"=>"T",
|
467
|
+
"local_resp"=>"F",
|
468
|
+
"missed_bytes"=>0,
|
469
|
+
"history"=>"^hCadcCfA"
|
470
|
+
},
|
471
|
+
"session_id"=>"CfVl4yNLJ4U3ptL12"
|
472
|
+
})
|
473
|
+
end
|
474
|
+
|
475
|
+
context 'with mac adresses' do
|
476
|
+
|
477
|
+
let(:message) do
|
478
|
+
"1128727435.633408 CHhAvVGS1DHFjwGM9 141.42.64.125 56730 125.190.109.199 80 tcp http 1.550793 98 9417 SF - - 0 ^hADdFaf 11 670 10 9945 - 00:d0:03:3b:f4:00 00:b0:c2:86:ec:00"
|
479
|
+
end
|
480
|
+
|
481
|
+
it 'matches' do
|
482
|
+
expect(grok).to include("timestamp" => "1128727435.633408")
|
483
|
+
expect(grok).to include("zeek" => hash_including("session_id" => "CHhAvVGS1DHFjwGM9"))
|
484
|
+
expect(grok).to include("network" => { "transport" => "tcp", "protocol" => "http" })
|
485
|
+
expect(grok).to include("source" => { "mac"=>"00:d0:03:3b:f4:00", "ip"=>"141.42.64.125", "port"=>56730, "packets"=>11, "bytes"=>670 })
|
486
|
+
expect(grok).to include("destination" => { "mac"=>"00:b0:c2:86:ec:00", "ip"=>"125.190.109.199", "port"=>80, "packets"=>10, "bytes"=>9945 })
|
487
|
+
|
488
|
+
expect(grok).to include("zeek"=>hash_including("connection" => hash_including("duration"=>1.550793)))
|
489
|
+
end
|
490
|
+
|
491
|
+
end
|
492
|
+
|
493
|
+
end
|
494
|
+
|
495
|
+
describe_pattern 'BRO_FILES', ['legacy', 'ecs-v1'] do
|
496
|
+
|
497
|
+
let(:message) do
|
498
|
+
"1362692527.009512 FakNcS1Jfe01uljb3 192.150.187.43 141.142.228.5 CXWv6p3arKYeMETxOg HTTP 0 MD5,SHA1 text/plain - 0.000263 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 - -"
|
499
|
+
end
|
500
|
+
|
501
|
+
it 'matches' do
|
502
|
+
if ecs_compatibility?
|
503
|
+
expect(grok).to include("timestamp"=>"1362692527.009512")
|
504
|
+
expect(grok).to include("server"=>{"ip"=>"192.150.187.43"})
|
505
|
+
expect(grok).to include("client"=>{"ip"=>"141.142.228.5"})
|
506
|
+
expect(grok).to include("zeek"=>{ "files" => {
|
507
|
+
"fuid"=>"FakNcS1Jfe01uljb3",
|
508
|
+
"session_ids"=>"CXWv6p3arKYeMETxOg",
|
509
|
+
"source"=>"HTTP", "depth"=>0, "analyzers"=>"MD5,SHA1",
|
510
|
+
"duration"=>0.000263,
|
511
|
+
"is_orig"=>"F",
|
512
|
+
"overflow_bytes"=>0, "missing_bytes"=>0, "seen_bytes"=>4705,
|
513
|
+
"timedout"=>"F"
|
514
|
+
}})
|
515
|
+
expect(grok).to include("file"=>{
|
516
|
+
"size"=>4705, "mime_type"=>"text/plain",
|
517
|
+
"hash"=>{"sha1"=>"1dd7ac0398df6cbc0696445a91ec681facf4dc47", "md5"=>"397168fd09991a0e712254df7bc639ac"}
|
518
|
+
})
|
519
|
+
else
|
520
|
+
expect(grok).to include("ts"=>"1362692527.009512", "fuid"=>"FakNcS1Jfe01uljb3")
|
521
|
+
expect(grok).to include("tx_hosts"=>"192.150.187.43", "rx_hosts"=>"141.142.228.5")
|
522
|
+
expect(grok).to include("conn_uids"=>"CXWv6p3arKYeMETxOg")
|
523
|
+
expect(grok).to include("source"=>"HTTP", "depth"=>"0", "analyzers"=>"MD5,SHA1", "mime_type"=>"text/plain")
|
524
|
+
expect(grok).to include("filename"=>"-")
|
525
|
+
expect(grok).to include("duration"=>"0.000263")
|
526
|
+
expect(grok).to include("local_orig"=>"-", "is_orig"=>"F",
|
527
|
+
"seen_bytes"=>"4705", "total_bytes"=>"4705", "missing_bytes"=>"0", "overflow_bytes"=>"0",
|
528
|
+
"timedout"=>"F",
|
529
|
+
"parent_fuid"=>"-",
|
530
|
+
"md5"=>"397168fd09991a0e712254df7bc639ac", "sha1"=>"1dd7ac0398df6cbc0696445a91ec681facf4dc47", "sha256"=>"-",
|
531
|
+
"extracted"=>"-"
|
532
|
+
)
|
533
|
+
end
|
534
|
+
end
|
535
|
+
|
536
|
+
context 'new (zeek) message format' do
|
537
|
+
|
538
|
+
let(:message) do
|
539
|
+
"1602576142.884704 Frp9wcpqbDl991Zh7 151.101.112.204 192.168.122.59 C3v3ce39xvI63Tn2E5 HTTP 0 SHA1,MD5 text/plain - 0.000000 F F 3707 51901 0 0 F - 995875a72487a52a657b94e3857ac4fe 00c45f6771d0006029a8ced68bf2a41ca3060e69 - - - -"
|
540
|
+
end
|
541
|
+
|
542
|
+
it 'matches (incorrectly) in legacy mode' do
|
543
|
+
unless ecs_compatibility?
|
544
|
+
expect(grok).to include("ts"=>"1602576142.884704", "fuid"=>"Frp9wcpqbDl991Zh7")
|
545
|
+
expect(grok).to include("source"=>"HTTP\t0\tSHA1,MD5") # BOGUS
|
546
|
+
end
|
547
|
+
end
|
548
|
+
|
549
|
+
it '~~does not match in ecs mode~~ matches (correctly) in ECS mode' do # since new format changes are additions at the end
|
550
|
+
if ecs_compatibility?
|
551
|
+
expect(grok).to include("timestamp"=>"1602576142.884704")
|
552
|
+
expect(grok).to include("server"=>{"ip"=>"151.101.112.204"})
|
553
|
+
expect(grok).to include("client"=>{"ip"=>"192.168.122.59"})
|
554
|
+
expect(grok).to include("zeek"=>{"files"=>{
|
555
|
+
"fuid"=>"Frp9wcpqbDl991Zh7",
|
556
|
+
"session_ids"=>"C3v3ce39xvI63Tn2E5",
|
557
|
+
"source"=>"HTTP", "depth"=>0, "analyzers"=>"SHA1,MD5",
|
558
|
+
"duration"=>0.0,
|
559
|
+
"local_orig"=>"F", "is_orig"=>"F",
|
560
|
+
"missing_bytes"=>0,
|
561
|
+
"timedout"=>"F",
|
562
|
+
"overflow_bytes"=>0, "seen_bytes"=>3707
|
563
|
+
}})
|
564
|
+
expect(grok).to include("file"=>{
|
565
|
+
"size"=>51901, "mime_type"=>"text/plain",
|
566
|
+
"hash"=>{"md5"=>"995875a72487a52a657b94e3857ac4fe", "sha1"=>"00c45f6771d0006029a8ced68bf2a41ca3060e69"}
|
567
|
+
})
|
568
|
+
end
|
569
|
+
end
|
570
|
+
|
571
|
+
end
|
572
|
+
|
573
|
+
end
|
574
|
+
|
575
|
+
describe_pattern 'ZEEK_FILES', ['ecs-v1'] do
|
576
|
+
|
577
|
+
let(:message) do
|
578
|
+
"1258867934.558264 F2xow8TIkvHG4Zz41 198.189.255.75 192.168.1.105 CHhAvVGS1DHFjwGM9 HTTP 0 EXTRACT - - 0.046240 - F 54229 605292323 4244449 0 T - - - - extract-1258867934.558264-HTTP-F2xow8TIkvHG4Zz41 T 4000"
|
579
|
+
end
|
580
|
+
|
581
|
+
it 'matches' do
|
582
|
+
expect(grok).to include("timestamp"=>"1258867934.558264")
|
583
|
+
expect(grok).to include("server"=>{"ip"=>'198.189.255.75'}) # tx_host
|
584
|
+
expect(grok).to include("client"=>{"ip"=>'192.168.1.105'}) # rx_host
|
585
|
+
expect(grok).to include("zeek"=>{"files"=>hash_including(
|
586
|
+
"fuid"=>"F2xow8TIkvHG4Zz41",
|
587
|
+
"session_ids"=>"CHhAvVGS1DHFjwGM9",
|
588
|
+
"source"=>"HTTP", "analyzers"=>"EXTRACT",
|
589
|
+
"timedout"=>"T",
|
590
|
+
|
591
|
+
"extracted"=>"extract-1258867934.558264-HTTP-F2xow8TIkvHG4Zz41",
|
592
|
+
"extracted_size"=>4000, "extracted_cutoff"=>"T"
|
593
|
+
)})
|
594
|
+
expect(grok).to include("file"=>{"size"=>605292323})
|
595
|
+
end
|
596
|
+
|
597
|
+
context 'multiple hosts' do
|
598
|
+
|
599
|
+
let(:message) do
|
600
|
+
# NOTE: Zeek sometimes treats set[ip/string] types as ',' separated other times as ' ' separated
|
601
|
+
"1258867934.558264 F2xow8TIkvHG4Zz41 10.1.1.2 10.1.1.1 192.168.1.105,192.168.1.10,192.168.1.11 - - 0 - - - 0.01234 - F - - - 0 T - - - - - - -"
|
602
|
+
end
|
603
|
+
|
604
|
+
it 'matches host(s)' do
|
605
|
+
expect(grok).to include("server"=>{"ip" => '10.1.1.2'}) # tx_host
|
606
|
+
expect(grok).to include("client"=>{"ip" => '192.168.1.105'}) # rx_host
|
607
|
+
expect(grok).to include("zeek"=>{"files" => hash_including("tx_hosts" => '10.1.1.2 10.1.1.1')})
|
608
|
+
expect(grok).to include("zeek"=>{"files" => hash_including("rx_hosts" => '192.168.1.105,192.168.1.10,192.168.1.11')})
|
609
|
+
end
|
610
|
+
|
611
|
+
end
|
612
|
+
|
613
|
+
end
|