logstash-patterns-core 4.1.2 → 4.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +119 -0
  3. data/Gemfile +8 -1
  4. data/LICENSE +199 -10
  5. data/README.md +12 -19
  6. data/lib/logstash/patterns/core.rb +11 -3
  7. data/logstash-patterns-core.gemspec +1 -1
  8. data/patterns/ecs-v1/aws +28 -0
  9. data/patterns/ecs-v1/bacula +53 -0
  10. data/patterns/ecs-v1/bind +13 -0
  11. data/patterns/ecs-v1/bro +30 -0
  12. data/patterns/ecs-v1/exim +26 -0
  13. data/patterns/ecs-v1/firewalls +111 -0
  14. data/patterns/ecs-v1/grok-patterns +95 -0
  15. data/patterns/ecs-v1/haproxy +40 -0
  16. data/patterns/ecs-v1/httpd +17 -0
  17. data/patterns/ecs-v1/java +34 -0
  18. data/patterns/ecs-v1/junos +13 -0
  19. data/patterns/ecs-v1/linux-syslog +16 -0
  20. data/patterns/{maven → ecs-v1/maven} +0 -0
  21. data/patterns/ecs-v1/mcollective +4 -0
  22. data/patterns/ecs-v1/mongodb +7 -0
  23. data/patterns/ecs-v1/nagios +124 -0
  24. data/patterns/ecs-v1/postgresql +2 -0
  25. data/patterns/ecs-v1/rails +13 -0
  26. data/patterns/ecs-v1/redis +3 -0
  27. data/patterns/ecs-v1/ruby +2 -0
  28. data/patterns/ecs-v1/squid +6 -0
  29. data/patterns/ecs-v1/zeek +33 -0
  30. data/patterns/{aws → legacy/aws} +1 -1
  31. data/patterns/{bacula → legacy/bacula} +5 -5
  32. data/patterns/legacy/bind +3 -0
  33. data/patterns/{bro → legacy/bro} +0 -0
  34. data/patterns/{exim → legacy/exim} +8 -2
  35. data/patterns/{firewalls → legacy/firewalls} +2 -2
  36. data/patterns/{grok-patterns → legacy/grok-patterns} +4 -4
  37. data/patterns/{haproxy → legacy/haproxy} +1 -1
  38. data/patterns/{httpd → legacy/httpd} +2 -2
  39. data/patterns/{java → legacy/java} +1 -3
  40. data/patterns/{junos → legacy/junos} +0 -0
  41. data/patterns/{linux-syslog → legacy/linux-syslog} +0 -0
  42. data/patterns/legacy/maven +1 -0
  43. data/patterns/{mcollective → legacy/mcollective} +0 -0
  44. data/patterns/{mcollective-patterns → legacy/mcollective-patterns} +0 -0
  45. data/patterns/{mongodb → legacy/mongodb} +0 -0
  46. data/patterns/{nagios → legacy/nagios} +1 -1
  47. data/patterns/{postgresql → legacy/postgresql} +0 -0
  48. data/patterns/{rails → legacy/rails} +0 -0
  49. data/patterns/{redis → legacy/redis} +0 -0
  50. data/patterns/{ruby → legacy/ruby} +0 -0
  51. data/patterns/legacy/squid +4 -0
  52. data/spec/patterns/aws_spec.rb +395 -0
  53. data/spec/patterns/bacula_spec.rb +367 -0
  54. data/spec/patterns/bind_spec.rb +92 -0
  55. data/spec/patterns/bro_spec.rb +613 -0
  56. data/spec/patterns/core_spec.rb +260 -15
  57. data/spec/patterns/exim_spec.rb +201 -0
  58. data/spec/patterns/firewalls_spec.rb +707 -66
  59. data/spec/patterns/haproxy_spec.rb +253 -28
  60. data/spec/patterns/httpd_spec.rb +248 -86
  61. data/spec/patterns/java_spec.rb +375 -0
  62. data/spec/patterns/junos_spec.rb +101 -0
  63. data/spec/patterns/mcollective_spec.rb +35 -0
  64. data/spec/patterns/mongodb_spec.rb +170 -33
  65. data/spec/patterns/nagios_spec.rb +299 -78
  66. data/spec/patterns/netscreen_spec.rb +123 -0
  67. data/spec/patterns/rails3_spec.rb +87 -29
  68. data/spec/patterns/redis_spec.rb +216 -140
  69. data/spec/patterns/shorewall_spec.rb +85 -74
  70. data/spec/patterns/squid_spec.rb +139 -0
  71. data/spec/patterns/syslog_spec.rb +266 -22
  72. data/spec/spec_helper.rb +83 -5
  73. metadata +70 -31
  74. data/patterns/bind +0 -3
  75. data/patterns/squid +0 -4
  76. data/spec/patterns/bro.rb +0 -126
  77. data/spec/patterns/s3_spec.rb +0 -173
@@ -0,0 +1,613 @@
1
+ # encoding: utf-8
2
+ require "spec_helper"
3
+ require "logstash/patterns/core"
4
+
5
+ describe_pattern "BRO_HTTP", ['legacy', 'ecs-v1'] do
6
+
7
+ let(:message) do # old BRO logging format
8
+ "1432555199.633017 COpk6E3vkURP8QQNKl 192.168.9.35 55281 178.236.7.146 80 4 POST www.amazon.it /xa/dealcontent/v2/GetDeals?nocache=1432555199326 http://www.amazon.it/ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 223 1859 200 OK - - - (empty) kares - - FrLEcY3AUPKdcYGf29 text/plain FOJpbGzIMh9syPxH8 text/plain"
9
+ end
10
+
11
+ it "matches a simple message" do
12
+ expect(pattern).to match(message)
13
+ end
14
+
15
+ it "generates the ts field" do
16
+ if ecs_compatibility?
17
+ expect(grok).to include("timestamp" => "1432555199.633017")
18
+ else
19
+ expect(grok).to include("ts" => "1432555199.633017")
20
+ end
21
+ end
22
+
23
+ it "generates the uid field" do
24
+ if ecs_compatibility?
25
+ expect(grok).to include("zeek" => hash_including("session_id" => "COpk6E3vkURP8QQNKl"))
26
+ else
27
+ expect(grok).to include("uid" => "COpk6E3vkURP8QQNKl")
28
+ end
29
+ end
30
+
31
+ it "generates the orig_ fields" do
32
+ if ecs_compatibility?
33
+ expect(grok).to include("source" => { "ip" => "192.168.9.35", "port" => 55281 })
34
+ else
35
+ expect(grok).to include("orig_h" => "192.168.9.35", "orig_p" => "55281")
36
+ end
37
+ end
38
+
39
+ it "generates the resp_ fields" do
40
+ if ecs_compatibility?
41
+ expect(grok).to include("destination" => { "ip" => "178.236.7.146", "port" => 80 })
42
+
43
+ else
44
+ expect(grok).to include("resp_h" => "178.236.7.146", "resp_p" => "80")
45
+ end
46
+ end
47
+
48
+ it "generates the trans_depth field" do
49
+ if ecs_compatibility?
50
+ expect(grok).to include("zeek" => hash_including("http" => hash_including("trans_depth" => 4)))
51
+ else
52
+ expect(grok).to include("trans_depth" => "4")
53
+ end
54
+ end
55
+
56
+ it "generates the method/referrer field" do
57
+ if ecs_compatibility?
58
+ expect(grok).to include("http" => hash_including("request" => hash_including("method" => "POST", "referrer" => "http://www.amazon.it/" )))
59
+ else
60
+ expect(grok).to include("method" => "POST")
61
+ expect(grok).to include("referrer" => "http://www.amazon.it/")
62
+ end
63
+ end
64
+
65
+ it "generates the domain/uri/referrer & username/password field" do
66
+ if ecs_compatibility?
67
+ expect(grok).to include("url" => hash_including("domain" => "www.amazon.it"))
68
+ expect(grok).to include("url" => hash_including("original" => "/xa/dealcontent/v2/GetDeals?nocache=1432555199326"))
69
+
70
+ expect(grok).to include("url" => hash_including("username" => "kares"))
71
+ expect(grok['url'].keys).to_not include("password")
72
+ else
73
+ expect(grok).to include("domain" => "www.amazon.it")
74
+ expect(grok).to include("uri" => "/xa/dealcontent/v2/GetDeals?nocache=1432555199326")
75
+
76
+ expect(grok).to include("username" => "kares")
77
+ expect(grok).to include("password" => "-")
78
+ end
79
+ end
80
+
81
+ it "generates the user_agent field" do
82
+ if ecs_compatibility?
83
+ expect(grok).to include("user_agent" => { "original" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36" })
84
+ else
85
+ expect(grok).to include("user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36")
86
+ end
87
+ end
88
+
89
+ it "generates the request_body_len/response_body_len fields" do
90
+ if ecs_compatibility?
91
+ expect(grok).to include("http" => hash_including("request" => hash_including("body" => { "bytes" => 223 })))
92
+ expect(grok).to include("http" => hash_including("response" => hash_including("body" => { "bytes" => 1859 })))
93
+ else
94
+ expect(grok).to include("request_body_len" => "223")
95
+ expect(grok).to include("response_body_len" => "1859")
96
+ end
97
+ end
98
+
99
+ it "generates the status_ fields" do
100
+ if ecs_compatibility?
101
+ expect(grok).to include("http" => hash_including("response" => hash_including("status_code" => 200)))
102
+ expect(grok).to include("zeek" => hash_including("http" => hash_including("status_msg" => "OK")))
103
+ else
104
+ expect(grok).to include("status_code" => "200", "status_msg" => "OK")
105
+ end
106
+ end
107
+
108
+ it "generates the info_ fields" do
109
+ if ecs_compatibility?
110
+ expect(grok['zeek']['http'].keys).to_not include("info_code", "info_msg")
111
+ else
112
+ expect(grok).to include("info_code" => "-", "info_msg" => "-")
113
+ end
114
+ end
115
+
116
+ it "generates the filename field" do
117
+ if ecs_compatibility?
118
+ expect(grok['zeek']['http'].keys).to_not include("filename")
119
+ else
120
+ expect(grok).to include("filename" => "-")
121
+ end
122
+ end
123
+
124
+ it "generates the bro_tags field" do
125
+ if ecs_compatibility?
126
+ expect(grok['zeek']['http'].keys).to_not include("tags")
127
+ else
128
+ expect(grok).to include("bro_tags" => "(empty)")
129
+ end
130
+ end
131
+
132
+ it "generates the proxied field" do
133
+ if ecs_compatibility?
134
+ expect(grok['zeek']['http'].keys).to_not include("proxied")
135
+ else
136
+ expect(grok).to include("proxied" => "-")
137
+ end
138
+ end
139
+
140
+ it "generates the orig_ fields" do
141
+ if ecs_compatibility?
142
+ expect(grok).to include("zeek" => hash_including("http" => hash_including("orig_fuids" => "FrLEcY3AUPKdcYGf29")))
143
+ expect(grok).to include("http" => hash_including("request" => hash_including("mime_type" => "text/plain")))
144
+ else
145
+ expect(grok).to include("orig_fuids" => "FrLEcY3AUPKdcYGf29")
146
+ expect(grok).to include("orig_mime_types" => "text/plain")
147
+ end
148
+ end
149
+
150
+ it "generates the resp_ fields" do
151
+ if ecs_compatibility?
152
+ expect(grok).to include("zeek" => hash_including("http" => hash_including("resp_fuids" => "FOJpbGzIMh9syPxH8")))
153
+ expect(grok).to include("http" => hash_including("response" => hash_including("mime_type" => "text/plain")))
154
+ else
155
+ expect(grok).to include("resp_fuids" => "FOJpbGzIMh9syPxH8")
156
+ expect(grok).to include("resp_mime_types" => "text/plain")
157
+ end
158
+ end
159
+
160
+ context '(zeek) updated log format' do
161
+
162
+ let(:message) do # ZEEK
163
+ '1602164975.587600 Ct73QY3M7T5dikxggf 192.168.122.59 55240 93.184.220.29 80 1 - - - - 1.1 - - 0 471 200 OK - - (empty) - - - - - - FPGXN33wAFL8MPKJXl - application/ocsp-response'
164
+ end
165
+
166
+ it 'matches in legacy mode' do
167
+ unless ecs_compatibility? # wrong but backwards compatibility
168
+ expect(grok).to include("domain" => "1.1") # due GREEDYDATA: "method" => "-\t-\t-\t-"
169
+ end
170
+ end
171
+
172
+ it 'no longer matches in ecs mode' do
173
+ expect(grok['tags']).to include("_grokparsefailure") if ecs_compatibility?
174
+ end
175
+
176
+ end
177
+
178
+ end
179
+
180
+ describe_pattern "ZEEK_HTTP", ['ecs-v1'] do
181
+
182
+ context "long message" do
183
+
184
+ let(:message) do
185
+ "1333458850.375568 ClEkJM2Vm5giqnMf4h 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf 1.1 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 - 0 56320 206 Partial Content - - (empty) - - - - - - FpmJd62pFQcZ3gcUgl - -"
186
+ end
187
+
188
+ it "matches" do
189
+ expect(grok).to include("timestamp"=>"1333458850.375568")
190
+ expect(grok).to include("zeek" => hash_including("session_id"=>"ClEkJM2Vm5giqnMf4h"))
191
+ expect(grok).to include("http" => {
192
+ "response" => { "body" =>{ "bytes" => 56320 }, "status_code"=>206 },
193
+ "request" => { "body" => { "bytes" => 0 }, "referrer"=>"http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf", "method"=>"GET"},
194
+ "version"=>"1.1"
195
+ })
196
+ expect(grok).to include("user_agent"=>{"original"=>"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11"})
197
+ expect(grok).to include("url"=>{
198
+ "domain"=>"o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com",
199
+ "original"=>"/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1"
200
+ })
201
+ expect(grok).to include("destination"=>{"port"=>80, "ip"=>"79.101.110.141"}, "source"=>{"port"=>1923, "ip"=>"10.131.47.185"})
202
+ expect(grok).to include("zeek" => hash_including("http"=>{"resp_fuids"=>"FpmJd62pFQcZ3gcUgl", "status_msg"=>"Partial Content", "trans_depth"=>1}))
203
+ end
204
+
205
+ end
206
+
207
+ context "sample message" do
208
+
209
+ let(:message) do
210
+ "1602165002.455618 CWGOypTfRypTC5C4g 192.168.122.59 44136 216.58.201.110 80 1 - - - - 1.1 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0 https://example.com 0 219 301 Moved Permanently - - FOO,BAR - - - - - - FeJ7iiVorMXoLlRK - text/html"
211
+ end
212
+
213
+ it "matches" do
214
+ expect(grok).to include("timestamp"=>"1602165002.455618")
215
+ expect(grok).to include("user_agent"=>{"original"=>"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0"})
216
+ expect(grok).to include("http" => {
217
+ "request" => { "body" => {"bytes" => 0 } },
218
+ "response" => { "status_code" => 301, "body" =>{ "bytes" => 219 }, "mime_type" => 'text/html' },
219
+ "version"=>"1.1"
220
+ })
221
+ expect(grok).to include("zeek" => {
222
+ "session_id" => "CWGOypTfRypTC5C4g",
223
+ "http" => {
224
+ "trans_depth"=>1,
225
+ "origin"=>"https://example.com",
226
+ "tags"=>"FOO,BAR",
227
+ "status_msg"=>"Moved Permanently",
228
+ "resp_fuids"=>"FeJ7iiVorMXoLlRK"}
229
+ })
230
+ end
231
+
232
+ end
233
+
234
+ context 'old (bro) message' do
235
+
236
+ let(:message) do
237
+ "1432555199.633017 COpk6E3vkURP8QQNKl 192.168.9.35 55281 178.236.7.146 80 4 POST www.amazon.it /xa/dealcontent/v2/GetDeals?nocache=1432555199326 http://www.amazon.it/ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 223 1859 200 OK - - - (empty) kares - - FrLEcY3AUPKdcYGf29 text/plain FOJpbGzIMh9syPxH8 text/plain"
238
+ end
239
+
240
+ it 'does not match' do
241
+ expect(grok['tags']).to include("_grokparsefailure") if ecs_compatibility?
242
+ end
243
+
244
+ end
245
+
246
+ context 'old empty (bro) message' do
247
+
248
+ let(:message) do # theoretically everything except these is optional:
249
+ "1432555199.633017 COpk6E3vkURP8QQNKl 192.168.9.35 55281 178.236.7.146 80 0 - - - - - - - - - - - - (empty) - - - - - - -"
250
+ end
251
+
252
+ it 'does not match' do
253
+ expect(grok['tags']).to include("_grokparsefailure") if ecs_compatibility?
254
+ end
255
+
256
+ end
257
+
258
+ end
259
+
260
+ describe_pattern 'BRO_DNS', ['legacy', 'ecs-v1'] do
261
+
262
+ let(:message) do # old BRO logging format
263
+ "1359565680.761790 UWkUyAuUGXf 192.168.6.10 53209 192.168.129.36 53 udp 41477 paypal.com 1 C_INTERNET 48 DNSKEY 0 NOERROR F F T F 1 - - F"
264
+ end
265
+
266
+ it 'matches' do
267
+ if ecs_compatibility?
268
+ expect(grok).to include("timestamp" => "1359565680.761790")
269
+ expect(grok).to include("zeek" => hash_including("session_id" => "UWkUyAuUGXf"))
270
+ expect(grok).to include("source" => { "ip"=>"192.168.6.10", "port"=>53209 }, "destination" => { "ip"=>"192.168.129.36", "port"=>53 })
271
+ expect(grok).to include("network" => { "transport" => "udp" })
272
+ expect(grok).to include("dns" => {
273
+ "id" => 41477,
274
+ "question" => { "name" => "paypal.com", "type" => "DNSKEY" },
275
+ "response_code" => "NOERROR",
276
+ })
277
+ expect(grok['zeek']).to include("dns" => hash_including("qclass" => 1, "qclass_name" => "C_INTERNET"))
278
+ expect(grok['zeek']).to include("dns" => hash_including("qtype" => 48)) # beats compatibility
279
+ expect(grok['zeek']).to include("dns" => hash_including("rcode" => 0)) # beats compatibility
280
+ # TODO :bool type-casting would be nice
281
+ expect(grok['zeek']).to include("dns" => hash_including("AA"=>"F", "TC"=>"F", "RD"=>"T", "RA"=>"F"))
282
+ expect(grok['zeek']).to include("dns" => hash_including("Z" => 1)) # beats drops this field
283
+ expect(grok['zeek']).to include("dns" => hash_including("rejected" => "F"))
284
+ expect(grok['zeek']['dns'].keys).to_not include 'TTLs', 'answers'
285
+ else
286
+ expect(grok).to include(
287
+ "ts"=>"1359565680.761790", "uid"=>"UWkUyAuUGXf",
288
+ "orig_h"=>"192.168.6.10", "orig_p"=>"53209",
289
+ "resp_h" => "192.168.129.36", "resp_p"=>"53",
290
+ "proto"=>"udp",
291
+ "trans_id"=>"41477",
292
+ "query"=>"paypal.com",
293
+ "qclass"=>"1", "qclass_name"=>"C_INTERNET",
294
+ "qtype"=>"48", "qtype_name"=>"DNSKEY",
295
+ "rcode"=>"0", "rcode_name"=>"NOERROR",
296
+ "AA"=>"F", "TC"=>"F",
297
+ "RD"=>"T", "RA"=>"F",
298
+ "Z"=>"1",
299
+ "answers"=>"-",
300
+ "TTLs"=>"-",
301
+ "rejected"=>"F",
302
+ )
303
+ end
304
+ end
305
+
306
+ context 'optional fields' do
307
+
308
+ let(:message) do
309
+ "1359565680.761790 UWkUyAuUGXf 192.168.6.10 53209 192.168.129.36 53 udp - - - - - - - - F F F F 0 - - -"
310
+ # AA/TC/RD/RA are optional with a F default, Z is optional with a 0 default
311
+ end
312
+
313
+ it 'matches (only) fields present' do
314
+ if ecs_compatibility?
315
+ expect(grok).to include("timestamp" => "1359565680.761790")
316
+ expect(grok).to include("source" => { "ip"=>"192.168.6.10", "port"=>53209 }, "destination" => { "ip"=>"192.168.129.36", "port"=>53 })
317
+ expect(grok).to include("network" => { "transport"=>"udp" })
318
+ expect(grok).to include("zeek" => { "session_id" => "UWkUyAuUGXf", "dns" => { "AA"=>"F", "TC"=>"F", "RD"=>"F", "RA"=>"F", "Z"=>0 } })
319
+ expect(grok.keys).to_not include('dns')
320
+ end
321
+ end
322
+
323
+ end
324
+
325
+ context '(zeek) updated log format' do
326
+
327
+ let(:message) do
328
+ "1359565680.761790 CHhAvVGS1DHFjwGM9 192.168.6.10 53209 192.168.129.36 53 udp 41477 0.075138 paypal.com 1 C_INTERNET 48 DNSKEY 0 NOERROR F F T T 1 DNSKEY 5,DNSKEY 5,RRSIG 48 paypal.com,RRSIG 48 paypal.com 455.000000,455.000000,455.000000,455.000000 F"
329
+ end
330
+
331
+ it 'no longer matches in ecs mode' do
332
+ expect(grok['tags']).to include("_grokparsefailure") if ecs_compatibility?
333
+ end
334
+
335
+ end
336
+
337
+ end
338
+
339
+ describe_pattern "ZEEK_DNS", ['ecs-v1'] do
340
+
341
+ let(:message) do
342
+ "1359565680.761790 CHhAvVGS1DHFjwGM9 192.168.6.10 53209 192.168.129.36 53 udp 41477 0.075138 paypal.com 1 C_INTERNET 48 DNSKEY 0 NOERROR F F T T 1 DNSKEY 5,DNSKEY 5,RRSIG 48 paypal.com,RRSIG 48 paypal.com 455.000000,455.000000,455.000000,455.000000 F"
343
+ end
344
+
345
+ it 'matches' do
346
+ expect(grok).to include("timestamp" => "1359565680.761790")
347
+ expect(grok).to include("destination"=>{"ip"=>"192.168.129.36", "port"=>53}, "source"=>{"ip"=>"192.168.6.10", "port"=>53209})
348
+ expect(grok).to include("network" => {"transport"=>"udp"})
349
+ expect(grok).to include("dns"=>{"question"=>{"type"=>"DNSKEY", "name"=>"paypal.com"}, "response_code"=>"NOERROR", "id"=>41477})
350
+ expect(grok).to include("zeek"=>{
351
+ "session_id"=>"CHhAvVGS1DHFjwGM9",
352
+ "dns"=>{
353
+ "rtt" => 0.075138 ,
354
+ "qclass"=>1, "qclass_name"=>"C_INTERNET", "qtype"=>48,
355
+ "rcode"=>0,
356
+ "RA"=>"T", "RD"=>"T", "TC"=>"F", "rejected"=>"F", "AA"=>"F", "Z"=>1,
357
+ "answers"=>"DNSKEY 5,DNSKEY 5,RRSIG 48 paypal.com,RRSIG 48 paypal.com",
358
+ "TTLs"=>"455.000000,455.000000,455.000000,455.000000"
359
+ }
360
+ })
361
+ end
362
+
363
+ end
364
+
365
+ describe_pattern 'BRO_CONN', ['legacy', 'ecs-v1'] do
366
+
367
+ let(:message) do
368
+ "1541350796.901974 C54zqz17PXuBv3HkLg 192.168.0.26 54855 54.85.115.89 443 tcp ssl 0.153642 1147 589 SF T 0 ShADadfF 7 1439 8 921 (empty)"
369
+ end
370
+
371
+ it 'matches' do
372
+ if ecs_compatibility?
373
+ expect(grok).to include("timestamp"=>"1541350796.901974")
374
+ expect(grok).to include("zeek" => hash_including("session_id" => "C54zqz17PXuBv3HkLg"))
375
+ expect(grok).to include("network"=>{"transport"=>"tcp", "protocol"=>"ssl"})
376
+ expect(grok).to include("source"=>{
377
+ "ip"=>"192.168.0.26", "port"=>54855,
378
+ "packets"=>7, "bytes"=>1439
379
+ })
380
+ expect(grok).to include("destination"=>{
381
+ "ip"=>"54.85.115.89", "port"=>443,
382
+ "packets"=>8, "bytes"=>921
383
+ })
384
+ expect(grok).to include("zeek" => hash_including(
385
+ "connection" => {
386
+ "duration"=>0.153642,
387
+ "orig_bytes"=>1147, "resp_bytes"=>589,
388
+ "state"=>"SF",
389
+ "local_orig"=>"T", "missed_bytes"=>0,
390
+ "history"=>"ShADadfF",
391
+ }
392
+ ))
393
+ else
394
+ expect(grok).to include(
395
+ "ts"=>"1541350796.901974", "uid"=>"C54zqz17PXuBv3HkLg",
396
+ "orig_h"=>"192.168.0.26", "orig_p"=>"54855",
397
+ "resp_h" => "54.85.115.89", "resp_p"=>"443",
398
+ "proto"=>"tcp",
399
+ "service"=>"ssl",
400
+ "duration"=>"0.153642",
401
+ "orig_bytes"=>"1147", "resp_bytes"=>"589",
402
+ "conn_state"=>"SF",
403
+ "local_orig"=>"T",
404
+ "missed_bytes"=>"0",
405
+ "history"=>"ShADadfF",
406
+ "orig_pkts"=>"7", "orig_ip_bytes"=>"1439",
407
+ "resp_pkts"=>"8", "resp_ip_bytes"=>"921",
408
+ "tunnel_parents"=>"(empty)",
409
+ )
410
+ end
411
+ end
412
+
413
+ context 'updated log format' do # up to version 3.3
414
+ let(:message) do
415
+ "1300475168.853899 C4J4Th3PJpwUYZZ6gc 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF - - 0 Dd 1 66 1 117 (empty)"
416
+ end
417
+
418
+ it 'matches' do
419
+ if ecs_compatibility?
420
+ expect(grok).to include("zeek" => hash_including("session_id" => "C4J4Th3PJpwUYZZ6gc"))
421
+ expect(grok).to include("network"=>{"transport"=>"udp", "protocol"=>"dns"})
422
+ expect(grok).to include("source"=>{
423
+ "ip"=>"141.142.220.118", "port"=>43927,
424
+ "packets"=>1, "bytes"=>66
425
+ })
426
+ expect(grok).to include("destination"=>{
427
+ "ip"=>"141.142.2.2", "port"=>53,
428
+ "packets"=>1, "bytes"=>117
429
+ })
430
+ expect(grok).to include("zeek" => hash_including(
431
+ "connection" => {
432
+ "duration"=>0.000435,
433
+ "orig_bytes"=>38, "resp_bytes"=>89,
434
+ "state"=>"SF",
435
+ "missed_bytes"=>0,
436
+ "history"=>"Dd",
437
+ }
438
+ ))
439
+ else
440
+ expect(grok).to include("ts"=>"1300475168.853899")
441
+ expect(grok).to include("conn_state"=>"SF\t-") # matches incorrectly
442
+ end
443
+ end
444
+ end
445
+
446
+ end
447
+
448
+ describe_pattern 'ZEEK_CONN', ['ecs-v1'] do
449
+
450
+ let(:message) do
451
+ "1602165198.216970 CfVl4yNLJ4U3ptL12 192.168.122.59 47340 143.204.201.102 443 tcp - 115.669583 0 6238 SHR T F 0 ^hCadcCfA 1 52 21 5091 -"
452
+ end
453
+
454
+ it 'matches' do
455
+ expect(grok).to include("timestamp" => "1602165198.216970")
456
+ expect(grok).to include("network" => { "transport" => "tcp" })
457
+ expect(grok).to include("source" => { "ip"=>"192.168.122.59", "port"=>47340, "packets"=>1, "bytes"=>52 })
458
+ expect(grok).to include("destination" => { "ip"=>"143.204.201.102", "port"=>443, "packets"=>21, "bytes"=>5091 })
459
+
460
+ expect(grok).to include("zeek"=>{
461
+ "connection"=>{
462
+ "duration"=>115.669583,
463
+ "orig_bytes"=>0,
464
+ "resp_bytes"=>6238,
465
+ "state"=>"SHR",
466
+ "local_orig"=>"T",
467
+ "local_resp"=>"F",
468
+ "missed_bytes"=>0,
469
+ "history"=>"^hCadcCfA"
470
+ },
471
+ "session_id"=>"CfVl4yNLJ4U3ptL12"
472
+ })
473
+ end
474
+
475
+ context 'with mac adresses' do
476
+
477
+ let(:message) do
478
+ "1128727435.633408 CHhAvVGS1DHFjwGM9 141.42.64.125 56730 125.190.109.199 80 tcp http 1.550793 98 9417 SF - - 0 ^hADdFaf 11 670 10 9945 - 00:d0:03:3b:f4:00 00:b0:c2:86:ec:00"
479
+ end
480
+
481
+ it 'matches' do
482
+ expect(grok).to include("timestamp" => "1128727435.633408")
483
+ expect(grok).to include("zeek" => hash_including("session_id" => "CHhAvVGS1DHFjwGM9"))
484
+ expect(grok).to include("network" => { "transport" => "tcp", "protocol" => "http" })
485
+ expect(grok).to include("source" => { "mac"=>"00:d0:03:3b:f4:00", "ip"=>"141.42.64.125", "port"=>56730, "packets"=>11, "bytes"=>670 })
486
+ expect(grok).to include("destination" => { "mac"=>"00:b0:c2:86:ec:00", "ip"=>"125.190.109.199", "port"=>80, "packets"=>10, "bytes"=>9945 })
487
+
488
+ expect(grok).to include("zeek"=>hash_including("connection" => hash_including("duration"=>1.550793)))
489
+ end
490
+
491
+ end
492
+
493
+ end
494
+
495
+ describe_pattern 'BRO_FILES', ['legacy', 'ecs-v1'] do
496
+
497
+ let(:message) do
498
+ "1362692527.009512 FakNcS1Jfe01uljb3 192.150.187.43 141.142.228.5 CXWv6p3arKYeMETxOg HTTP 0 MD5,SHA1 text/plain - 0.000263 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 - -"
499
+ end
500
+
501
+ it 'matches' do
502
+ if ecs_compatibility?
503
+ expect(grok).to include("timestamp"=>"1362692527.009512")
504
+ expect(grok).to include("server"=>{"ip"=>"192.150.187.43"})
505
+ expect(grok).to include("client"=>{"ip"=>"141.142.228.5"})
506
+ expect(grok).to include("zeek"=>{ "files" => {
507
+ "fuid"=>"FakNcS1Jfe01uljb3",
508
+ "session_ids"=>"CXWv6p3arKYeMETxOg",
509
+ "source"=>"HTTP", "depth"=>0, "analyzers"=>"MD5,SHA1",
510
+ "duration"=>0.000263,
511
+ "is_orig"=>"F",
512
+ "overflow_bytes"=>0, "missing_bytes"=>0, "seen_bytes"=>4705,
513
+ "timedout"=>"F"
514
+ }})
515
+ expect(grok).to include("file"=>{
516
+ "size"=>4705, "mime_type"=>"text/plain",
517
+ "hash"=>{"sha1"=>"1dd7ac0398df6cbc0696445a91ec681facf4dc47", "md5"=>"397168fd09991a0e712254df7bc639ac"}
518
+ })
519
+ else
520
+ expect(grok).to include("ts"=>"1362692527.009512", "fuid"=>"FakNcS1Jfe01uljb3")
521
+ expect(grok).to include("tx_hosts"=>"192.150.187.43", "rx_hosts"=>"141.142.228.5")
522
+ expect(grok).to include("conn_uids"=>"CXWv6p3arKYeMETxOg")
523
+ expect(grok).to include("source"=>"HTTP", "depth"=>"0", "analyzers"=>"MD5,SHA1", "mime_type"=>"text/plain")
524
+ expect(grok).to include("filename"=>"-")
525
+ expect(grok).to include("duration"=>"0.000263")
526
+ expect(grok).to include("local_orig"=>"-", "is_orig"=>"F",
527
+ "seen_bytes"=>"4705", "total_bytes"=>"4705", "missing_bytes"=>"0", "overflow_bytes"=>"0",
528
+ "timedout"=>"F",
529
+ "parent_fuid"=>"-",
530
+ "md5"=>"397168fd09991a0e712254df7bc639ac", "sha1"=>"1dd7ac0398df6cbc0696445a91ec681facf4dc47", "sha256"=>"-",
531
+ "extracted"=>"-"
532
+ )
533
+ end
534
+ end
535
+
536
+ context 'new (zeek) message format' do
537
+
538
+ let(:message) do
539
+ "1602576142.884704 Frp9wcpqbDl991Zh7 151.101.112.204 192.168.122.59 C3v3ce39xvI63Tn2E5 HTTP 0 SHA1,MD5 text/plain - 0.000000 F F 3707 51901 0 0 F - 995875a72487a52a657b94e3857ac4fe 00c45f6771d0006029a8ced68bf2a41ca3060e69 - - - -"
540
+ end
541
+
542
+ it 'matches (incorrectly) in legacy mode' do
543
+ unless ecs_compatibility?
544
+ expect(grok).to include("ts"=>"1602576142.884704", "fuid"=>"Frp9wcpqbDl991Zh7")
545
+ expect(grok).to include("source"=>"HTTP\t0\tSHA1,MD5") # BOGUS
546
+ end
547
+ end
548
+
549
+ it '~~does not match in ecs mode~~ matches (correctly) in ECS mode' do # since new format changes are additions at the end
550
+ if ecs_compatibility?
551
+ expect(grok).to include("timestamp"=>"1602576142.884704")
552
+ expect(grok).to include("server"=>{"ip"=>"151.101.112.204"})
553
+ expect(grok).to include("client"=>{"ip"=>"192.168.122.59"})
554
+ expect(grok).to include("zeek"=>{"files"=>{
555
+ "fuid"=>"Frp9wcpqbDl991Zh7",
556
+ "session_ids"=>"C3v3ce39xvI63Tn2E5",
557
+ "source"=>"HTTP", "depth"=>0, "analyzers"=>"SHA1,MD5",
558
+ "duration"=>0.0,
559
+ "local_orig"=>"F", "is_orig"=>"F",
560
+ "missing_bytes"=>0,
561
+ "timedout"=>"F",
562
+ "overflow_bytes"=>0, "seen_bytes"=>3707
563
+ }})
564
+ expect(grok).to include("file"=>{
565
+ "size"=>51901, "mime_type"=>"text/plain",
566
+ "hash"=>{"md5"=>"995875a72487a52a657b94e3857ac4fe", "sha1"=>"00c45f6771d0006029a8ced68bf2a41ca3060e69"}
567
+ })
568
+ end
569
+ end
570
+
571
+ end
572
+
573
+ end
574
+
575
+ describe_pattern 'ZEEK_FILES', ['ecs-v1'] do
576
+
577
+ let(:message) do
578
+ "1258867934.558264 F2xow8TIkvHG4Zz41 198.189.255.75 192.168.1.105 CHhAvVGS1DHFjwGM9 HTTP 0 EXTRACT - - 0.046240 - F 54229 605292323 4244449 0 T - - - - extract-1258867934.558264-HTTP-F2xow8TIkvHG4Zz41 T 4000"
579
+ end
580
+
581
+ it 'matches' do
582
+ expect(grok).to include("timestamp"=>"1258867934.558264")
583
+ expect(grok).to include("server"=>{"ip"=>'198.189.255.75'}) # tx_host
584
+ expect(grok).to include("client"=>{"ip"=>'192.168.1.105'}) # rx_host
585
+ expect(grok).to include("zeek"=>{"files"=>hash_including(
586
+ "fuid"=>"F2xow8TIkvHG4Zz41",
587
+ "session_ids"=>"CHhAvVGS1DHFjwGM9",
588
+ "source"=>"HTTP", "analyzers"=>"EXTRACT",
589
+ "timedout"=>"T",
590
+
591
+ "extracted"=>"extract-1258867934.558264-HTTP-F2xow8TIkvHG4Zz41",
592
+ "extracted_size"=>4000, "extracted_cutoff"=>"T"
593
+ )})
594
+ expect(grok).to include("file"=>{"size"=>605292323})
595
+ end
596
+
597
+ context 'multiple hosts' do
598
+
599
+ let(:message) do
600
+ # NOTE: Zeek sometimes treats set[ip/string] types as ',' separated other times as ' ' separated
601
+ "1258867934.558264 F2xow8TIkvHG4Zz41 10.1.1.2 10.1.1.1 192.168.1.105,192.168.1.10,192.168.1.11 - - 0 - - - 0.01234 - F - - - 0 T - - - - - - -"
602
+ end
603
+
604
+ it 'matches host(s)' do
605
+ expect(grok).to include("server"=>{"ip" => '10.1.1.2'}) # tx_host
606
+ expect(grok).to include("client"=>{"ip" => '192.168.1.105'}) # rx_host
607
+ expect(grok).to include("zeek"=>{"files" => hash_including("tx_hosts" => '10.1.1.2 10.1.1.1')})
608
+ expect(grok).to include("zeek"=>{"files" => hash_including("rx_hosts" => '192.168.1.105,192.168.1.10,192.168.1.11')})
609
+ end
610
+
611
+ end
612
+
613
+ end