logstash-patterns-core 4.2.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +98 -0
  3. data/Gemfile +3 -0
  4. data/README.md +11 -18
  5. data/lib/logstash/patterns/core.rb +11 -3
  6. data/logstash-patterns-core.gemspec +1 -1
  7. data/patterns/ecs-v1/aws +28 -0
  8. data/patterns/ecs-v1/bacula +53 -0
  9. data/patterns/ecs-v1/bind +13 -0
  10. data/patterns/ecs-v1/bro +30 -0
  11. data/patterns/ecs-v1/exim +26 -0
  12. data/patterns/ecs-v1/firewalls +111 -0
  13. data/patterns/ecs-v1/grok-patterns +95 -0
  14. data/patterns/ecs-v1/haproxy +40 -0
  15. data/patterns/ecs-v1/httpd +17 -0
  16. data/patterns/ecs-v1/java +34 -0
  17. data/patterns/ecs-v1/junos +13 -0
  18. data/patterns/ecs-v1/linux-syslog +16 -0
  19. data/patterns/{maven → ecs-v1/maven} +0 -0
  20. data/patterns/ecs-v1/mcollective +4 -0
  21. data/patterns/ecs-v1/mongodb +7 -0
  22. data/patterns/ecs-v1/nagios +124 -0
  23. data/patterns/ecs-v1/postgresql +2 -0
  24. data/patterns/ecs-v1/rails +13 -0
  25. data/patterns/ecs-v1/redis +3 -0
  26. data/patterns/ecs-v1/ruby +2 -0
  27. data/patterns/ecs-v1/squid +6 -0
  28. data/patterns/ecs-v1/zeek +33 -0
  29. data/patterns/{aws → legacy/aws} +1 -1
  30. data/patterns/{bacula → legacy/bacula} +5 -5
  31. data/patterns/legacy/bind +3 -0
  32. data/patterns/{bro → legacy/bro} +0 -0
  33. data/patterns/{exim → legacy/exim} +8 -2
  34. data/patterns/{firewalls → legacy/firewalls} +2 -2
  35. data/patterns/{grok-patterns → legacy/grok-patterns} +0 -0
  36. data/patterns/{haproxy → legacy/haproxy} +0 -0
  37. data/patterns/{httpd → legacy/httpd} +1 -1
  38. data/patterns/{java → legacy/java} +0 -0
  39. data/patterns/{junos → legacy/junos} +0 -0
  40. data/patterns/{linux-syslog → legacy/linux-syslog} +0 -0
  41. data/patterns/legacy/maven +1 -0
  42. data/patterns/{mcollective → legacy/mcollective} +0 -0
  43. data/patterns/{mcollective-patterns → legacy/mcollective-patterns} +0 -0
  44. data/patterns/{mongodb → legacy/mongodb} +0 -0
  45. data/patterns/{nagios → legacy/nagios} +0 -0
  46. data/patterns/{postgresql → legacy/postgresql} +0 -0
  47. data/patterns/{rails → legacy/rails} +0 -0
  48. data/patterns/{redis → legacy/redis} +0 -0
  49. data/patterns/{ruby → legacy/ruby} +0 -0
  50. data/patterns/legacy/squid +4 -0
  51. data/spec/patterns/aws_spec.rb +395 -0
  52. data/spec/patterns/bacula_spec.rb +367 -0
  53. data/spec/patterns/bind_spec.rb +78 -0
  54. data/spec/patterns/bro_spec.rb +613 -0
  55. data/spec/patterns/core_spec.rb +51 -9
  56. data/spec/patterns/exim_spec.rb +201 -0
  57. data/spec/patterns/firewalls_spec.rb +669 -66
  58. data/spec/patterns/haproxy_spec.rb +246 -38
  59. data/spec/patterns/httpd_spec.rb +215 -94
  60. data/spec/patterns/java_spec.rb +357 -27
  61. data/spec/patterns/junos_spec.rb +101 -0
  62. data/spec/patterns/mcollective_spec.rb +35 -0
  63. data/spec/patterns/mongodb_spec.rb +170 -33
  64. data/spec/patterns/nagios_spec.rb +296 -79
  65. data/spec/patterns/netscreen_spec.rb +123 -0
  66. data/spec/patterns/rails3_spec.rb +87 -29
  67. data/spec/patterns/redis_spec.rb +157 -121
  68. data/spec/patterns/shorewall_spec.rb +85 -74
  69. data/spec/patterns/squid_spec.rb +139 -0
  70. data/spec/patterns/syslog_spec.rb +266 -22
  71. data/spec/spec_helper.rb +80 -6
  72. metadata +64 -28
  73. data/patterns/bind +0 -3
  74. data/patterns/squid +0 -4
  75. data/spec/patterns/bro.rb +0 -126
  76. data/spec/patterns/s3_spec.rb +0 -173
@@ -2,19 +2,60 @@
2
2
  require "spec_helper"
3
3
  require "logstash/patterns/core"
4
4
 
5
- describe "HAPROXY" do
5
+ describe_pattern "HAPROXYHTTP", ['legacy', 'ecs-v1'] do
6
6
 
7
- let(:haproxyhttp_pattern) { "HAPROXYHTTP" }
7
+ context "log line from raw syslog line" do
8
8
 
9
- context "Parsing HAPROXY log line from raw syslog line" do
9
+ let(:message) do
10
+ 'Dec 9 13:01:26 localhost haproxy[28029]: 127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"'
11
+ end
12
+
13
+ it "matches" do
14
+ if ecs_compatibility?
15
+ expect(subject).to include("timestamp"=>"Dec 9 13:01:26")
16
+ expect(subject).to include("host"=>{"hostname"=>"localhost"})
17
+ expect(subject).to include("process"=>{"pid"=>28029, "name"=>"haproxy"})
18
+ expect(subject).to include("source"=>{"port"=>39759, "address"=>"127.0.0.1", "bytes"=>83285})
19
+ expect(subject).to include("haproxy" => hash_including("request_date"=>"09/Dec/2013:12:59:46.633"))
20
+ expect(subject).to include("haproxy" => hash_including("frontend_name"=>"loadbalancer", "backend_name"=>"default", "server_name"=>"instance8"))
21
+ expect(subject).to include("haproxy" => hash_including(
22
+ "total_waiting_time_ms"=>51536, "connection_wait_time_ms"=>1, "total_time_ms"=>"99627",
23
+ "http" => hash_including("request"=>hash_including("time_wait_ms"=>0, "time_wait_without_data_ms"=>48082))
24
+ ))
25
+ expect(subject).to include("http" => hash_including("response"=>{"status_code"=>200}))
26
+
27
+ expect(subject).to include("haproxy" => hash_including("termination_state"=>"----"))
28
+
29
+ expect(subject).to include("haproxy" => hash_including("connections"=>{"active"=>87, "frontend"=>87, "backend"=>87, "server"=>1, "retries"=>0}))
30
+ expect(subject).to include("haproxy" => hash_including("backend_queue"=>67, "server_queue"=>0))
10
31
 
11
- let(:value) { 'Dec 9 13:01:26 localhost haproxy[28029]: 127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"' }
12
- subject { grok_match(haproxyhttp_pattern, value) }
32
+ expect(subject).to include("http" => hash_including("request" => {"method"=>'GET'}, "version" => '1.1'))
13
33
 
14
- it { should include("program" => "haproxy") }
15
- it { should include("client_ip" => "127.0.0.1") }
16
- it { should include("http_verb" => "GET") }
17
- it { should include("server_name" => "instance8") }
34
+ expect(subject).to include("url" => { "original"=>"/path/to/image", "path"=>"/path/to/image" })
35
+ else
36
+ expect(subject).to include("syslog_timestamp" => "Dec 9 13:01:26")
37
+ expect(subject).to include("syslog_server" => "localhost")
38
+ expect(subject).to include("http_request" => "/path/to/image", "http_status_code" => "200", "http_verb" => "GET", "http_version" => "1.1")
39
+ expect(subject).to include("program" => "haproxy")
40
+ expect(subject).to include("client_ip" => "127.0.0.1")
41
+ expect(subject).to include("http_verb" => "GET")
42
+ expect(subject).to include("server_name" => "instance8")
43
+ end
44
+ end
45
+
46
+ it "has no captured cookies" do
47
+ if ecs_compatibility?
48
+ expect((subject['haproxy']['http']['request'] || {}).keys).to_not include('captured_cookie')
49
+ expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_cookie')
50
+ end
51
+ end
52
+
53
+ it "includes header captures" do
54
+ if ecs_compatibility?
55
+ expect((subject['haproxy']['http'])).to include('request' => hash_including('captured_headers' => '77.24.148.74'))
56
+ expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_headers')
57
+ end
58
+ end
18
59
 
19
60
  it "generates a message field" do
20
61
  expect(subject["message"]).to include("loadbalancer default/instance8")
@@ -22,55 +63,222 @@ describe "HAPROXY" do
22
63
 
23
64
  end
24
65
 
25
- context "Parsing HAPROXY log line from raw syslog line with ISO8601 timestamp" do
66
+ context "log line (without headers) from raw syslog line with ISO8601 timestamp" do
26
67
 
27
- let(:value) { '2015-08-26T02:09:48+02:00 localhost haproxy[28029]: 127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"' }
28
- subject { grok_match(haproxyhttp_pattern, value) }
68
+ let(:message) do
69
+ '2015-08-26T02:09:48+02:00 localhost haproxy[14389]: 5.196.2.38:39527 [03/Nov/2015:06:25:25.105] services~ def/api 4599/0/0/428/5027 304 320 - - ---- 1/1/0/1/0 0/0 "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1"'
70
+ end
29
71
 
30
- it { should include("program" => "haproxy") }
31
- it { should include("client_ip" => "127.0.0.1") }
32
- it { should include("http_verb" => "GET") }
33
- it { should include("server_name" => "instance8") }
72
+ it "matches" do
73
+ if ecs_compatibility?
74
+ expect(subject).to include("timestamp"=>"2015-08-26T02:09:48+02:00")
75
+ expect(subject).to include("host"=>{"hostname"=>"localhost"})
76
+ expect(subject).to include("process"=>{"pid"=>14389, "name"=>"haproxy"})
34
77
 
35
- it "generates a message field" do
36
- expect(subject["message"]).to include("loadbalancer default/instance8")
78
+ expect(subject).to include("haproxy" => hash_including("connections"=>{"active"=>1, "frontend"=>1, "backend"=>0, "server"=>1, "retries"=>0}))
79
+ expect(subject).to include("haproxy" => hash_including("backend_queue"=>0, "server_queue"=>0))
80
+
81
+ expect(subject).to include("haproxy" => hash_including("frontend_name"=>"services~"))
82
+
83
+ expect(subject).to include("http"=>{"response"=>{"status_code"=>304}, "version"=>"1.1", "request"=>{"method"=>"GET"}})
84
+ expect(subject).to include("url"=>hash_including("path"=>"/component---src-pages-index-js-4b15624544f97cf0bb8f.js"))
85
+ else
86
+ expect(subject).to include("program" => "haproxy")
87
+ expect(subject).to include("client_ip" => "5.196.2.38")
88
+ expect(subject).to include("http_verb" => "GET")
89
+ expect(subject).to include("server_name" => "api")
90
+ end
91
+ end
92
+
93
+ it "has no header captures" do
94
+ if ecs_compatibility?
95
+ expect((subject['haproxy']['http']['request'] || {}).keys).to_not include('captured_headers')
96
+ expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_headers')
97
+ end
37
98
  end
38
99
 
39
100
  end
40
101
 
41
- let(:haproxyhttpbase_pattern) { "HAPROXYHTTPBASE" }
102
+ context 'log line with both request/response headers' do
42
103
 
43
- context "Parsing HAPROXY log line without syslog specific enteries. This mimics an event coming from a syslog input." do
104
+ let(:message) do
105
+ 'Jul 30 09:03:52 home.host haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} "GET http://192.168.0.12:8080/serv/login.php?lang=en&profile=2 HTTP/1.1"'
106
+ end
44
107
 
45
- let(:value) { '127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"' }
46
- subject { grok_match(haproxyhttpbase_pattern, value) }
108
+ it "matches" do
109
+ if ecs_compatibility?
110
+ expect(subject).to include("timestamp"=>"Jul 30 09:03:52")
111
+ expect(subject).to include("host"=>{"hostname"=>"home.host"})
47
112
 
48
- # Assume 'program' would be matched by the syslog input.
49
- it { should include("client_ip" => "127.0.0.1") }
50
- it { should include("http_verb" => "GET") }
51
- it { should include("server_name" => "instance8") }
113
+ expect(subject).to include("haproxy" => hash_including("frontend_name"=>"incoming~"))
52
114
 
53
- it "generates a message field" do
54
- expect(subject["message"]).to include("loadbalancer default/instance8")
115
+ expect(subject).to include("http"=>{"response"=>{"status_code"=>304}, "version"=>"1.1", "request"=>{"method"=>"GET"}})
116
+ expect(subject).to include("url"=>hash_including("scheme"=>"http", "domain"=>"192.168.0.12", "port"=>8080,
117
+ "path"=>"/serv/login.php", "query"=>"lang=en&profile=2",
118
+ "original"=>"http://192.168.0.12:8080/serv/login.php?lang=en&profile=2"))
119
+ else
120
+ expect(subject).to include("client_ip" => "1.2.3.4")
121
+ expect(subject).to include("http_verb" => "GET")
122
+ end
123
+ end
124
+
125
+ it "has header captures" do
126
+ if ecs_compatibility?
127
+ expect((subject['haproxy']['http']['request'])).to include('captured_headers' => 'docs.example.internal||')
128
+ expect((subject['haproxy']['http']['response'])).to include('captured_headers' => '|||')
129
+ end
55
130
  end
56
131
 
57
132
  end
58
133
 
59
- context "Parsing HAPROXY log line that is truncated and thus not ending with a double quote or HTTP version." do
134
+ context 'BADREQ/NOSRV log line' do
60
135
 
61
- let(:value) { 'Jul 31 22:20:22 loadbalancer haproxy[1190]: 203.0.113.54:59968 [31/Jul/2017:22:20:22.447] loadbalancer default/instance8 135/0/1/19/156 200 1015 - - --VR 8/8/0/0/0 0/0 "GET /path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss'}
62
- subject { grok_match(haproxyhttpbase_pattern, value)}
136
+ let(:message) do
137
+ 'Jul 18 17:05:30 localhost haproxy[8247]: 188.223.50.7:51940 [18/Jul/2011:17:05:24.339] http_proxy_ads http_proxy_ads/<NOSRV> -1/-1/-1/-1/6001 408 212 - - cR-- 100/89/0/0/0 0/0 "<BADREQ>"'
138
+ end
63
139
 
64
- it { should include("client_ip" => "203.0.113.54") }
65
- it { should include("http_verb" => "GET") }
66
- it { should include("server_name" => "instance8") }
67
- it { should include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss") }
68
- it { should_not have_key("http_version") }
140
+ it "matches" do
141
+ if ecs_compatibility?
142
+ expect(subject).to include("timestamp"=>"Jul 18 17:05:30")
69
143
 
70
- it "generates a message field" do
71
- expect(subject["message"]).to include("loadbalancer default/instance8")
144
+ expect(subject).to include("haproxy" => hash_including("frontend_name"=>"http_proxy_ads"))
145
+ expect(subject).to include("haproxy" => hash_including("backend_name"=>"http_proxy_ads"))
146
+ expect(subject['haproxy'].keys).to_not include('server_name')
147
+ expect(subject).to include("http"=>{"response"=>{"status_code"=>408}})
148
+ expect(subject['haproxy'].keys).to_not include("total_waiting_time_ms", "connection_wait_time_ms")
149
+ expect(subject).to include("haproxy" => hash_including("total_time_ms"=>"6001"))
150
+ expect(subject).to include("source" => hash_including("bytes"=>212))
151
+ expect(subject).to include("haproxy" => hash_including("termination_state"=>"cR--"))
152
+ expect(subject.keys).to_not include("url")
153
+ else
154
+ expect(subject).to include("backend_name"=>"http_proxy_ads", "frontend_name"=>"http_proxy_ads", "server_name"=>"<NOSRV>")
155
+ expect(subject).to include("http_status_code"=>"408")
156
+ expect(subject).to include("time_backend_connect"=>"-1", "time_queue"=>"-1", "time_backend_response"=>"-1")
157
+ expect(subject).to include("captured_request_cookie"=>"-", "captured_response_cookie"=>"-")
158
+ expect(subject).to include("bytes_read"=>"212")
159
+ expect(subject).to include("termination_state"=>"cR--")
160
+ end
72
161
  end
73
162
 
74
163
  end
75
164
 
76
165
  end
166
+
167
+ describe_pattern "HAPROXYHTTPBASE", ['ecs-v1', 'legacy'] do
168
+
169
+ context "log line without syslog specific entries" do # This mimics an event coming from a syslog input.
170
+
171
+ let(:message) do
172
+ '127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/+99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET / HTTP/1.1"'
173
+ end
174
+
175
+ it 'matches' do
176
+ if ecs_compatibility?
177
+ expect(subject).to include("source"=>{"port"=>39759, "address"=>"127.0.0.1", "bytes"=>83285})
178
+ expect(subject).to include("haproxy"=>hash_including("server_queue"=>0,
179
+ "http"=>{
180
+ "request"=>{"time_wait_ms"=>0, "captured_headers"=>"77.24.148.74", "time_wait_without_data_ms"=>48082}
181
+ },
182
+
183
+ # NOTE: this is why we do not type-cast to :int
184
+ # a '+' sign is prepended before the value, indicating that the final one will be larger
185
+ "total_time_ms" => "+99627"
186
+ ))
187
+ expect(subject).to include("url"=>{"path"=>"/", "original"=>"/"})
188
+ else
189
+ # Assume 'program' would be matched by the syslog input.
190
+ expect(subject).to include("client_ip" => "127.0.0.1")
191
+ expect(subject).to include("server_name" => "instance8")
192
+ expect(subject).to include("http_verb" => "GET", "http_request"=>"/", "http_version" => '1.1')
193
+ expect(subject).to include("time_duration" => "+99627")
194
+ end
195
+ end
196
+
197
+ end
198
+
199
+ context "(incomplete) log line that is truncated and thus not ending with a double quote or HTTP version" do
200
+
201
+ let(:message) do
202
+ 'Jul 31 22:20:22 loadbalancer haproxy[1190]: 203.0.113.54:59968 [31/Jul/2017:22:20:22.447] loadbalancer default/instance8 135/0/1/19/156 200 1015 - - --VR 8/8/0/0/0 0/0 "GET /path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss'
203
+ end
204
+
205
+ it 'matches' do
206
+ if ecs_compatibility?
207
+ # due compatibility with the legacy pattern we match the incomplete "REQUEST LINE ... (wout the ending '"')
208
+ expect(subject).to include("http"=>{"response"=>{"status_code"=>200}, "request"=>{"method"=>"GET"}})
209
+ expect(subject).to include("url"=>hash_including("original"=>"/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss"))
210
+ else
211
+ expect(subject).to include("client_ip" => "203.0.113.54")
212
+ expect(subject).to include("http_verb" => "GET")
213
+ expect(subject).to include("server_name" => "instance8")
214
+ expect(subject).to include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss")
215
+ expect(subject).to_not have_key("http_version")
216
+ end
217
+ end
218
+
219
+ end
220
+
221
+
222
+ context "connect line with host:port url" do
223
+
224
+ let(:message) do
225
+ 'Nov 4 08:32:18 debian10 haproxy[3666]: 127.0.0.1:34500 [04/Nov/2020:08:32:18.194] samplefrontend backendnodes/node1 0/0/0/0/0 405 501 - - ---- 1/1/0/1/0 0/0 "CONNECT localhost:8080 HTTP/1.1"'
226
+ end
227
+
228
+ it 'matches' do
229
+ if ecs_compatibility?
230
+ expect(subject).to include("http"=>hash_including("request"=>{"method"=>"CONNECT"}))
231
+ expect(subject).to include("url"=>{"port"=>8080, "original"=>"localhost:8080", "domain"=>"localhost"})
232
+ else
233
+ expect(subject).to include("http_verb" => "CONNECT")
234
+ expect(subject).to include("http_host" => "localhost:8080")
235
+ end
236
+ end
237
+
238
+ end
239
+
240
+ end
241
+
242
+ describe_pattern "HAPROXYTCP", ['legacy', 'ecs-v1'] do
243
+
244
+ let(:message) do
245
+ 'Sep 20 15:44:23 127.0.0.1 haproxy[25457]: 127.0.0.1:40962 [20/Sep/2018:15:44:23.285] main app/<NOSRV> -1/-1/1 212 SC 1/1/0/0/0 0/0'
246
+ end
247
+
248
+ it 'matches' do
249
+ if ecs_compatibility?
250
+ expect(subject).to include(
251
+ "timestamp"=>"Sep 20 15:44:23",
252
+ "host"=>{"hostname"=>"127.0.0.1"},
253
+ "process"=>{"pid"=>25457, "name"=>"haproxy"},
254
+ "source"=>{"port"=>40962, "address"=>"127.0.0.1", "bytes"=>212},
255
+ "haproxy"=>{
256
+ "request_date"=>"20/Sep/2018:15:44:23.285",
257
+ "frontend_name"=>"main", "backend_name"=>"app",
258
+ "total_time_ms"=>"1",
259
+ "termination_state"=>"SC",
260
+ "connections"=>{"active"=>1, "backend"=>0, "retries"=>0, "server"=>0, "frontend"=>1},
261
+ "server_queue"=>0, "backend_queue"=>0
262
+ })
263
+ else
264
+ expect(subject).to include(
265
+ "syslog_timestamp"=>"Sep 20 15:44:23",
266
+ "syslog_server"=>"127.0.0.1",
267
+ "program"=>"haproxy", "pid"=>"25457",
268
+ "client_ip"=>"127.0.0.1", "client_port"=>"40962",
269
+ "accept_date"=>"20/Sep/2018:15:44:23.285",
270
+ "frontend_name"=>"main",
271
+ "backend_name"=>"app",
272
+ "server_name"=>"<NOSRV>",
273
+ "time_backend_connect"=>"-1",
274
+ "time_queue"=>"-1",
275
+ "time_duration"=>"1",
276
+ "bytes_read"=>"212",
277
+ "termination_state"=>"SC",
278
+ "actconn"=>"1", "feconn"=>"1", "beconn"=>"0", "backend_queue"=>"0", "retries"=>"0",
279
+ "srv_queue"=>"0", "srvconn"=>"0",
280
+ )
281
+ end
282
+ end
283
+
284
+ end
@@ -2,30 +2,51 @@
2
2
  require "spec_helper"
3
3
  require "logstash/patterns/core"
4
4
 
5
- describe "HTTPD_COMBINEDLOG" do
6
-
7
- let(:pattern) { 'HTTPD_COMBINEDLOG' }
8
- let(:grok) { grok_match(pattern, message) }
5
+ describe_pattern "HTTPD_COMBINEDLOG", ['legacy', 'ecs-v1'] do
9
6
 
10
7
  context "typical test case" do
11
8
 
12
9
  let(:message) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'}
13
10
 
14
11
  it "matches" do
15
- expect(grok).to include(
16
- 'clientip' => '83.149.9.216',
17
- 'verb' => 'GET',
18
- 'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
19
- 'httpversion' => '1.1',
20
- 'response' => '200',
21
- 'bytes' => '203023',
22
- 'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
23
- 'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
24
- )
12
+ if ecs_compatibility?
13
+ expect(grok).to include(
14
+ "http" => {
15
+ "request" => {
16
+ "method" => "GET",
17
+ "referrer" => "http://semicomplete.com/presentations/logstash-monitorama-2013/"
18
+ },
19
+ "response" => {
20
+ "body" => { "bytes" => 203023 },
21
+ "status_code" => 200
22
+ },
23
+ "version"=>"1.1"
24
+ },
25
+ "source" => { "address" => "83.149.9.216" },
26
+ "url" => { "original" => "/presentations/logstash-monitorama-2013/images/kibana-search.png" },
27
+ "user_agent" => { "original" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" }
28
+ )
29
+ else
30
+ expect(grok).to include(
31
+ 'clientip' => '83.149.9.216',
32
+ 'verb' => 'GET',
33
+ 'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
34
+ 'httpversion' => '1.1',
35
+ 'response' => '200',
36
+ 'bytes' => '203023',
37
+ 'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
38
+ 'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
39
+ )
40
+ end
25
41
  end
26
42
 
27
43
  it "does not capture 'null' fields" do
28
- expect(grok).to include('auth' => '-', 'ident' => '-')
44
+ if ecs_compatibility?
45
+ expect(grok.keys).to_not include('user') # 'user' => 'name'
46
+ expect(grok.keys).to_not include('apache') # apache.access.user.identity
47
+ else
48
+ expect(grok).to include('auth' => '-', 'ident' => '-')
49
+ end
29
50
  end
30
51
 
31
52
  end
@@ -35,7 +56,11 @@ describe "HTTPD_COMBINEDLOG" do
35
56
  let(:message) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'}
36
57
 
37
58
  it "gets captured" do
38
- expect(grok).to include("auth" => "username@example.com")
59
+ if ecs_compatibility?
60
+ expect(grok).to include("user" => { 'name' => "username@example.com" })
61
+ else
62
+ expect(grok).to include("auth" => "username@example.com")
63
+ end
39
64
  end
40
65
 
41
66
  end
@@ -45,121 +70,212 @@ describe "HTTPD_COMBINEDLOG" do
45
70
  let(:message) { '83.149.9.216 - a.user [11/Jan/2020:23:05:27 +0100] "OPTIONS /remote.php/ HTTP/1.1" - 7908 "-" "monitoring-client (v2.2)"' }
46
71
 
47
72
  it 'matches' do
48
- expect(grok).to include("verb" => "OPTIONS", 'request' => '/remote.php/', 'httpversion' => '1.1', "bytes" => '7908')
73
+ if ecs_compatibility?
74
+ expect(grok).to include("http" => hash_including("response" => hash_including("body" => { "bytes" => 7908 })))
75
+ expect(grok).to include("http" => hash_including("request" => { "method" => "OPTIONS" }, "version" => "1.1"))
76
+ expect(grok).to include(
77
+ "url" => { "original" => "/remote.php/" },
78
+ "user_agent" => { "original" => "monitoring-client (v2.2)" }
79
+ )
80
+ else
81
+ expect(grok).to include("verb" => "OPTIONS", 'request' => '/remote.php/', 'httpversion' => '1.1', "bytes" => '7908')
82
+ end
49
83
  end
50
84
 
51
85
  it 'does not capture optional response code' do
52
- expect(grok.keys).to_not include("response")
86
+ if ecs_compatibility?
87
+ expect(grok['http']['response'].keys).to_not include("status_code")
88
+ else
89
+ expect(grok.keys).to_not include("response")
90
+ end
53
91
  end
54
92
 
55
93
  end
56
94
 
57
95
  end
58
96
 
59
- describe "HTTPD_ERRORLOG" do
60
-
61
- let(:pattern) { 'HTTPD_ERRORLOG' }
62
- let(:grok) { grok_match(pattern, message) }
97
+ describe_pattern "HTTPD_ERRORLOG", ['legacy', 'ecs-v1'] do
63
98
 
64
- context "matches a full httpd 2.4 message" do
99
+ context "a full httpd 2.4 message" do
65
100
  let(:message) do
66
101
  "[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product"
67
102
  end
68
- it "generates the fields" do
69
103
 
70
- expect(grok).to include(
71
- 'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
72
- 'module' => 'proxy_fcgi',
73
- 'loglevel' => 'error',
74
- 'pid' => '28787',
75
- 'tid' => '140169587934976',
76
- 'proxy_errorcode' => '70008',
77
- 'proxy_message' => 'Partial results are valid but processing is incomplete',
78
- 'clientip' => '58.13.45.166',
79
- 'clientport' => '59307',
80
- 'errorcode' => 'AH01075',
81
- 'message' => [ message, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ],
82
- )
104
+ it "generates the fields" do
105
+ expect(grok).to include('timestamp' => 'Mon Aug 31 09:30:48.958285 2015')
106
+ if ecs_compatibility?
107
+ expect(grok).to include("log" => { "level" => "error" })
108
+ expect(grok).to include("process" => { "pid" => 28787, "thread" => { "id" => 140169587934976 } })
109
+ expect(grok).to include("source" => { "address"=>"58.13.45.166", "port" => 59307 })
110
+ expect(grok).to include("error" => { "code" => 'AH01075' })
111
+ expect(grok).to include("apache" => { "error" => {
112
+ "module" => "proxy_fcgi",
113
+ "proxy" => { "error" => { "code" => '70008', "message" => "Partial results are valid but processing is incomplete" }}}
114
+ })
115
+ else
116
+ expect(grok).to include(
117
+ 'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
118
+ 'module' => 'proxy_fcgi',
119
+ 'loglevel' => 'error',
120
+ 'pid' => '28787',
121
+ 'tid' => '140169587934976',
122
+ 'proxy_errorcode' => '70008',
123
+ 'proxy_message' => 'Partial results are valid but processing is incomplete',
124
+ 'clientip' => '58.13.45.166',
125
+ 'clientport' => '59307',
126
+ 'errorcode' => 'AH01075'
127
+ )
128
+ end
129
+ expect(grok).to include('message' => [ message, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ])
83
130
  end
84
131
  end
85
132
 
86
- context "HTTPD_ERRORLOG", "matches a httpd 2.2 log message" do
133
+ context "a httpd 2.2 log message" do
87
134
  let(:message) do
88
135
  "[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com"
89
136
  end
137
+
90
138
  it "generates the fields" do
91
- expect(grok).to include(
92
- 'timestamp' => 'Mon Aug 31 16:27:04 2015',
93
- 'loglevel' => 'error',
94
- 'clientip' => '10.17.42.3',
95
- 'message' => [ message, 'Premature end of script headers: example.com' ]
96
- )
139
+ if ecs_compatibility?
140
+ expect(grok).to include(
141
+ "timestamp"=>"Mon Aug 31 16:27:04 2015",
142
+ "log"=>{"level"=>"error"},
143
+ "source"=>{"address"=>"10.17.42.3"})
144
+ expect(grok.keys).to_not include("error") # error.code
145
+ else
146
+ expect(grok).to include(
147
+ 'timestamp' => 'Mon Aug 31 16:27:04 2015',
148
+ 'loglevel' => 'error',
149
+ 'clientip' => '10.17.42.3'
150
+ )
151
+ expect(grok.keys).to_not include('errorcode')
152
+ end
153
+ expect(grok).to include('message' => [ message, 'Premature end of script headers: example.com' ])
97
154
  end
98
155
  end
99
156
 
100
- context "HTTPD_ERRORLOG", "a short httpd 2.4 message" do
157
+ context "a short httpd 2.4 message" do
101
158
  let(:value1) {
102
159
  "[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'"
103
160
  }
104
161
  it "generates the fields" do
105
- expect(grok_match(subject, value1)).to include(
106
- 'timestamp' => 'Mon Aug 31 07:15:38.664897 2015',
107
- 'module' => 'proxy_fcgi',
108
- 'loglevel' => 'error',
109
- 'pid' => '28786',
110
- 'tid' => '140169629898496',
111
- 'clientip' => '81.139.1.34',
112
- 'clientport' => '52042',
113
- 'errorcode' => 'AH01071',
114
- 'message' => [ value1, "Got error 'Primary script unknown\n'" ]
115
- )
162
+ match_result = grok_match(pattern, value1)
163
+ expect(match_result).to include('timestamp' => 'Mon Aug 31 07:15:38.664897 2015')
164
+ if ecs_compatibility?
165
+ expect(match_result).to include(
166
+ "apache"=>{"error"=>{"module"=>"proxy_fcgi"}},
167
+ "log"=>{"level"=>"error"},
168
+ "process"=>{"pid"=>28786, "thread"=>{"id"=>140169629898496}},
169
+ "source"=>{"address"=>"81.139.1.34", "port"=>52042},
170
+ "error"=>{"code"=>"AH01071"},
171
+ )
172
+ else
173
+ expect(match_result).to include(
174
+ 'module' => 'proxy_fcgi',
175
+ 'loglevel' => 'error',
176
+ 'pid' => '28786',
177
+ 'tid' => '140169629898496',
178
+ 'clientip' => '81.139.1.34',
179
+ 'clientport' => '52042',
180
+ 'errorcode' => 'AH01071'
181
+ )
182
+ end
183
+ expect(match_result).to include('message' => [ value1, "Got error 'Primary script unknown\n'" ])
116
184
  end
117
185
 
118
186
  let(:value2) {
119
187
  "[Thu Apr 27 10:39:46.719636 2017] [php7:notice] [pid 17] [client 10.255.0.3:49580] Test error log record"
120
188
  }
121
- it "generates the fields" do
122
- expect(grok_match(subject, value2)).to include(
123
- 'timestamp' => 'Thu Apr 27 10:39:46.719636 2017',
124
- 'module' => 'php7',
125
- 'loglevel' => 'notice',
126
- 'pid' => '17',
127
- 'clientip' => '10.255.0.3',
128
- 'clientport' => '49580',
129
- 'message' => [ value2, "Test error log record" ]
130
- )
189
+ it "generates the fields" do
190
+ match_result = grok_match(pattern, value2)
191
+ expect(match_result).to include('timestamp' => 'Thu Apr 27 10:39:46.719636 2017')
192
+ if ecs_compatibility?
193
+ expect(match_result).to include(
194
+ "apache"=>{"error"=>{"module"=>"php7"}},
195
+ "log"=>{"level"=>"notice"},
196
+ "process"=>{"pid"=>17},
197
+ "source"=>{"port"=>49580, "address"=>"10.255.0.3"}
198
+ )
199
+ else
200
+ expect(match_result).to include(
201
+ 'module' => 'php7',
202
+ 'loglevel' => 'notice',
203
+ 'pid' => '17',
204
+ 'clientip' => '10.255.0.3',
205
+ 'clientport' => '49580'
206
+ )
207
+ end
208
+ expect(match_result).to include('message' => [ value2, "Test error log record" ])
131
209
  end
132
210
  end
133
211
 
134
- context "HTTPD_ERRORLOG", "a httpd 2.4 restart message" do
212
+ context "a httpd 2.4 restart message" do
135
213
  let(:value1) {
136
214
  "[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations"
137
215
  }
138
216
  it "generates the fields" do
139
- expect(grok_match(subject, value1)).to include(
140
- 'timestamp' => 'Mon Aug 31 06:29:47.406518 2015',
141
- 'module' => 'mpm_event',
142
- 'loglevel' => 'notice',
143
- 'pid' => '24968',
144
- 'tid' => '140169861986176',
145
- 'errorcode' => 'AH00489',
146
- 'message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ]
147
- )
217
+ match_result = grok_match(pattern, value1)
218
+ expect(match_result).to include('timestamp' => 'Mon Aug 31 06:29:47.406518 2015')
219
+ if ecs_compatibility?
220
+ expect(match_result).to include(
221
+ "apache"=>{"error"=>{"module"=>"mpm_event"}},
222
+ "log"=>{"level"=>"notice"},
223
+ "process"=>{"pid"=>24968, "thread"=>{"id"=>140169861986176}},
224
+ "error"=>{"code"=>"AH00489"}
225
+ )
226
+
227
+ else
228
+ expect(match_result).to include(
229
+ 'module' => 'mpm_event',
230
+ 'loglevel' => 'notice',
231
+ 'pid' => '24968',
232
+ 'tid' => '140169861986176',
233
+ 'errorcode' => 'AH00489'
234
+ )
235
+ end
236
+ expect(match_result).to include('message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ])
148
237
  end
149
238
 
150
239
  let(:value2) {
151
240
  "[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'"
152
241
  }
153
242
  it "generates the fields" do
154
- expect(grok_match(subject, value2)).to include(
155
- 'timestamp' => 'Mon Aug 31 06:29:47.406530 2015',
156
- 'module' => 'core',
157
- 'loglevel' => 'notice',
158
- 'pid' => '24968',
159
- 'tid' => '140169861986176',
160
- 'errorcode' => 'AH00094',
161
- 'message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ]
162
- )
243
+ match_result = grok_match(pattern, value2)
244
+ expect(match_result).to include('timestamp' => 'Mon Aug 31 06:29:47.406530 2015')
245
+ if ecs_compatibility?
246
+ expect(match_result).to include(
247
+ "apache"=>{"error"=>{"module"=>"core"}},
248
+ "log"=>{"level"=>"notice"},
249
+ "process"=>{"pid"=>24968, "thread"=>{"id"=>140169861986176}},
250
+ "error"=>{"code"=>"AH00094"}
251
+ )
252
+ else
253
+ expect(match_result).to include(
254
+ 'module' => 'core',
255
+ 'loglevel' => 'notice',
256
+ 'pid' => '24968',
257
+ 'tid' => '140169861986176',
258
+ 'errorcode' => 'AH00094'
259
+ )
260
+ end
261
+ expect(match_result).to include('message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ])
262
+ end
263
+ end
264
+
265
+ context "a httpd 2.4 message witout module" do
266
+ let(:message) do
267
+ "[Tue Apr 14 14:27:52.605084 2020] [:error] [pid 5688] [client 192.168.10.110:8196] script '/login/wp-login.php' not found or unable to stat"
268
+ end
269
+
270
+ it "matches" do
271
+ expect(grok).to include('timestamp' => 'Tue Apr 14 14:27:52.605084 2020')
272
+ if ecs_compatibility?
273
+ expect(grok).to include("log"=>{"level" => "error"})
274
+ expect(grok).to include("process"=>{"pid" => 5688})
275
+ expect( ((grok['apache'] || {})['error'] || {}).keys ).to_not include('module')
276
+ else
277
+ expect(grok).to include('loglevel' => 'error', 'pid' => '5688')
278
+ end
163
279
  end
164
280
  end
165
281
 
@@ -169,15 +285,20 @@ describe "HTTPD_ERRORLOG" do
169
285
  end
170
286
 
171
287
  it 'matches imperfectly (legacy)' do
172
- expect(grok).to include({
173
- "timestamp"=>"Fri Feb 01 22:03:08.319124 2019",
174
- "module"=>"authz_core",
175
- "loglevel"=>"debug",
176
- "pid"=>"9",
177
- "tid"=>"140597881775872",
178
- "errorcode"=>"mod_authz_core.c(820)",
179
- "message"=>[message, "[client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted"]
180
- })
288
+ if ecs_compatibility?
289
+ pending
290
+ raise NotImplementedError, "TODO: would be nice to 'improve' matching on these debug logs as well"
291
+ else
292
+ expect(grok).to include({
293
+ "timestamp"=>"Fri Feb 01 22:03:08.319124 2019",
294
+ "module"=>"authz_core",
295
+ "loglevel"=>"debug",
296
+ "pid"=>"9",
297
+ "tid"=>"140597881775872",
298
+ "errorcode"=>"mod_authz_core.c(820)",
299
+ "message"=>[message, "[client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted"]
300
+ })
301
+ end
181
302
  end
182
303
  end
183
304