logstash-patterns-core 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +98 -0
- data/Gemfile +3 -0
- data/README.md +11 -18
- data/lib/logstash/patterns/core.rb +11 -3
- data/logstash-patterns-core.gemspec +1 -1
- data/patterns/ecs-v1/aws +28 -0
- data/patterns/ecs-v1/bacula +53 -0
- data/patterns/ecs-v1/bind +13 -0
- data/patterns/ecs-v1/bro +30 -0
- data/patterns/ecs-v1/exim +26 -0
- data/patterns/ecs-v1/firewalls +111 -0
- data/patterns/ecs-v1/grok-patterns +95 -0
- data/patterns/ecs-v1/haproxy +40 -0
- data/patterns/ecs-v1/httpd +17 -0
- data/patterns/ecs-v1/java +34 -0
- data/patterns/ecs-v1/junos +13 -0
- data/patterns/ecs-v1/linux-syslog +16 -0
- data/patterns/{maven → ecs-v1/maven} +0 -0
- data/patterns/ecs-v1/mcollective +4 -0
- data/patterns/ecs-v1/mongodb +7 -0
- data/patterns/ecs-v1/nagios +124 -0
- data/patterns/ecs-v1/postgresql +2 -0
- data/patterns/ecs-v1/rails +13 -0
- data/patterns/ecs-v1/redis +3 -0
- data/patterns/ecs-v1/ruby +2 -0
- data/patterns/ecs-v1/squid +6 -0
- data/patterns/ecs-v1/zeek +33 -0
- data/patterns/{aws → legacy/aws} +1 -1
- data/patterns/{bacula → legacy/bacula} +5 -5
- data/patterns/legacy/bind +3 -0
- data/patterns/{bro → legacy/bro} +0 -0
- data/patterns/{exim → legacy/exim} +8 -2
- data/patterns/{firewalls → legacy/firewalls} +2 -2
- data/patterns/{grok-patterns → legacy/grok-patterns} +0 -0
- data/patterns/{haproxy → legacy/haproxy} +0 -0
- data/patterns/{httpd → legacy/httpd} +1 -1
- data/patterns/{java → legacy/java} +0 -0
- data/patterns/{junos → legacy/junos} +0 -0
- data/patterns/{linux-syslog → legacy/linux-syslog} +0 -0
- data/patterns/legacy/maven +1 -0
- data/patterns/{mcollective → legacy/mcollective} +0 -0
- data/patterns/{mcollective-patterns → legacy/mcollective-patterns} +0 -0
- data/patterns/{mongodb → legacy/mongodb} +0 -0
- data/patterns/{nagios → legacy/nagios} +0 -0
- data/patterns/{postgresql → legacy/postgresql} +0 -0
- data/patterns/{rails → legacy/rails} +0 -0
- data/patterns/{redis → legacy/redis} +0 -0
- data/patterns/{ruby → legacy/ruby} +0 -0
- data/patterns/legacy/squid +4 -0
- data/spec/patterns/aws_spec.rb +395 -0
- data/spec/patterns/bacula_spec.rb +367 -0
- data/spec/patterns/bind_spec.rb +78 -0
- data/spec/patterns/bro_spec.rb +613 -0
- data/spec/patterns/core_spec.rb +51 -9
- data/spec/patterns/exim_spec.rb +201 -0
- data/spec/patterns/firewalls_spec.rb +669 -66
- data/spec/patterns/haproxy_spec.rb +246 -38
- data/spec/patterns/httpd_spec.rb +215 -94
- data/spec/patterns/java_spec.rb +357 -27
- data/spec/patterns/junos_spec.rb +101 -0
- data/spec/patterns/mcollective_spec.rb +35 -0
- data/spec/patterns/mongodb_spec.rb +170 -33
- data/spec/patterns/nagios_spec.rb +296 -79
- data/spec/patterns/netscreen_spec.rb +123 -0
- data/spec/patterns/rails3_spec.rb +87 -29
- data/spec/patterns/redis_spec.rb +157 -121
- data/spec/patterns/shorewall_spec.rb +85 -74
- data/spec/patterns/squid_spec.rb +139 -0
- data/spec/patterns/syslog_spec.rb +266 -22
- data/spec/spec_helper.rb +80 -6
- metadata +64 -28
- data/patterns/bind +0 -3
- data/patterns/squid +0 -4
- data/spec/patterns/bro.rb +0 -126
- data/spec/patterns/s3_spec.rb +0 -173
@@ -2,19 +2,60 @@
|
|
2
2
|
require "spec_helper"
|
3
3
|
require "logstash/patterns/core"
|
4
4
|
|
5
|
-
|
5
|
+
describe_pattern "HAPROXYHTTP", ['legacy', 'ecs-v1'] do
|
6
6
|
|
7
|
-
|
7
|
+
context "log line from raw syslog line" do
|
8
8
|
|
9
|
-
|
9
|
+
let(:message) do
|
10
|
+
'Dec 9 13:01:26 localhost haproxy[28029]: 127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"'
|
11
|
+
end
|
12
|
+
|
13
|
+
it "matches" do
|
14
|
+
if ecs_compatibility?
|
15
|
+
expect(subject).to include("timestamp"=>"Dec 9 13:01:26")
|
16
|
+
expect(subject).to include("host"=>{"hostname"=>"localhost"})
|
17
|
+
expect(subject).to include("process"=>{"pid"=>28029, "name"=>"haproxy"})
|
18
|
+
expect(subject).to include("source"=>{"port"=>39759, "address"=>"127.0.0.1", "bytes"=>83285})
|
19
|
+
expect(subject).to include("haproxy" => hash_including("request_date"=>"09/Dec/2013:12:59:46.633"))
|
20
|
+
expect(subject).to include("haproxy" => hash_including("frontend_name"=>"loadbalancer", "backend_name"=>"default", "server_name"=>"instance8"))
|
21
|
+
expect(subject).to include("haproxy" => hash_including(
|
22
|
+
"total_waiting_time_ms"=>51536, "connection_wait_time_ms"=>1, "total_time_ms"=>"99627",
|
23
|
+
"http" => hash_including("request"=>hash_including("time_wait_ms"=>0, "time_wait_without_data_ms"=>48082))
|
24
|
+
))
|
25
|
+
expect(subject).to include("http" => hash_including("response"=>{"status_code"=>200}))
|
26
|
+
|
27
|
+
expect(subject).to include("haproxy" => hash_including("termination_state"=>"----"))
|
28
|
+
|
29
|
+
expect(subject).to include("haproxy" => hash_including("connections"=>{"active"=>87, "frontend"=>87, "backend"=>87, "server"=>1, "retries"=>0}))
|
30
|
+
expect(subject).to include("haproxy" => hash_including("backend_queue"=>67, "server_queue"=>0))
|
10
31
|
|
11
|
-
|
12
|
-
subject { grok_match(haproxyhttp_pattern, value) }
|
32
|
+
expect(subject).to include("http" => hash_including("request" => {"method"=>'GET'}, "version" => '1.1'))
|
13
33
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
34
|
+
expect(subject).to include("url" => { "original"=>"/path/to/image", "path"=>"/path/to/image" })
|
35
|
+
else
|
36
|
+
expect(subject).to include("syslog_timestamp" => "Dec 9 13:01:26")
|
37
|
+
expect(subject).to include("syslog_server" => "localhost")
|
38
|
+
expect(subject).to include("http_request" => "/path/to/image", "http_status_code" => "200", "http_verb" => "GET", "http_version" => "1.1")
|
39
|
+
expect(subject).to include("program" => "haproxy")
|
40
|
+
expect(subject).to include("client_ip" => "127.0.0.1")
|
41
|
+
expect(subject).to include("http_verb" => "GET")
|
42
|
+
expect(subject).to include("server_name" => "instance8")
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
it "has no captured cookies" do
|
47
|
+
if ecs_compatibility?
|
48
|
+
expect((subject['haproxy']['http']['request'] || {}).keys).to_not include('captured_cookie')
|
49
|
+
expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_cookie')
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
it "includes header captures" do
|
54
|
+
if ecs_compatibility?
|
55
|
+
expect((subject['haproxy']['http'])).to include('request' => hash_including('captured_headers' => '77.24.148.74'))
|
56
|
+
expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_headers')
|
57
|
+
end
|
58
|
+
end
|
18
59
|
|
19
60
|
it "generates a message field" do
|
20
61
|
expect(subject["message"]).to include("loadbalancer default/instance8")
|
@@ -22,55 +63,222 @@ describe "HAPROXY" do
|
|
22
63
|
|
23
64
|
end
|
24
65
|
|
25
|
-
context "
|
66
|
+
context "log line (without headers) from raw syslog line with ISO8601 timestamp" do
|
26
67
|
|
27
|
-
let(:
|
28
|
-
|
68
|
+
let(:message) do
|
69
|
+
'2015-08-26T02:09:48+02:00 localhost haproxy[14389]: 5.196.2.38:39527 [03/Nov/2015:06:25:25.105] services~ def/api 4599/0/0/428/5027 304 320 - - ---- 1/1/0/1/0 0/0 "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1"'
|
70
|
+
end
|
29
71
|
|
30
|
-
it
|
31
|
-
|
32
|
-
|
33
|
-
|
72
|
+
it "matches" do
|
73
|
+
if ecs_compatibility?
|
74
|
+
expect(subject).to include("timestamp"=>"2015-08-26T02:09:48+02:00")
|
75
|
+
expect(subject).to include("host"=>{"hostname"=>"localhost"})
|
76
|
+
expect(subject).to include("process"=>{"pid"=>14389, "name"=>"haproxy"})
|
34
77
|
|
35
|
-
|
36
|
-
|
78
|
+
expect(subject).to include("haproxy" => hash_including("connections"=>{"active"=>1, "frontend"=>1, "backend"=>0, "server"=>1, "retries"=>0}))
|
79
|
+
expect(subject).to include("haproxy" => hash_including("backend_queue"=>0, "server_queue"=>0))
|
80
|
+
|
81
|
+
expect(subject).to include("haproxy" => hash_including("frontend_name"=>"services~"))
|
82
|
+
|
83
|
+
expect(subject).to include("http"=>{"response"=>{"status_code"=>304}, "version"=>"1.1", "request"=>{"method"=>"GET"}})
|
84
|
+
expect(subject).to include("url"=>hash_including("path"=>"/component---src-pages-index-js-4b15624544f97cf0bb8f.js"))
|
85
|
+
else
|
86
|
+
expect(subject).to include("program" => "haproxy")
|
87
|
+
expect(subject).to include("client_ip" => "5.196.2.38")
|
88
|
+
expect(subject).to include("http_verb" => "GET")
|
89
|
+
expect(subject).to include("server_name" => "api")
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
it "has no header captures" do
|
94
|
+
if ecs_compatibility?
|
95
|
+
expect((subject['haproxy']['http']['request'] || {}).keys).to_not include('captured_headers')
|
96
|
+
expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_headers')
|
97
|
+
end
|
37
98
|
end
|
38
99
|
|
39
100
|
end
|
40
101
|
|
41
|
-
|
102
|
+
context 'log line with both request/response headers' do
|
42
103
|
|
43
|
-
|
104
|
+
let(:message) do
|
105
|
+
'Jul 30 09:03:52 home.host haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} "GET http://192.168.0.12:8080/serv/login.php?lang=en&profile=2 HTTP/1.1"'
|
106
|
+
end
|
44
107
|
|
45
|
-
|
46
|
-
|
108
|
+
it "matches" do
|
109
|
+
if ecs_compatibility?
|
110
|
+
expect(subject).to include("timestamp"=>"Jul 30 09:03:52")
|
111
|
+
expect(subject).to include("host"=>{"hostname"=>"home.host"})
|
47
112
|
|
48
|
-
|
49
|
-
it { should include("client_ip" => "127.0.0.1") }
|
50
|
-
it { should include("http_verb" => "GET") }
|
51
|
-
it { should include("server_name" => "instance8") }
|
113
|
+
expect(subject).to include("haproxy" => hash_including("frontend_name"=>"incoming~"))
|
52
114
|
|
53
|
-
|
54
|
-
|
115
|
+
expect(subject).to include("http"=>{"response"=>{"status_code"=>304}, "version"=>"1.1", "request"=>{"method"=>"GET"}})
|
116
|
+
expect(subject).to include("url"=>hash_including("scheme"=>"http", "domain"=>"192.168.0.12", "port"=>8080,
|
117
|
+
"path"=>"/serv/login.php", "query"=>"lang=en&profile=2",
|
118
|
+
"original"=>"http://192.168.0.12:8080/serv/login.php?lang=en&profile=2"))
|
119
|
+
else
|
120
|
+
expect(subject).to include("client_ip" => "1.2.3.4")
|
121
|
+
expect(subject).to include("http_verb" => "GET")
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
it "has header captures" do
|
126
|
+
if ecs_compatibility?
|
127
|
+
expect((subject['haproxy']['http']['request'])).to include('captured_headers' => 'docs.example.internal||')
|
128
|
+
expect((subject['haproxy']['http']['response'])).to include('captured_headers' => '|||')
|
129
|
+
end
|
55
130
|
end
|
56
131
|
|
57
132
|
end
|
58
133
|
|
59
|
-
context
|
134
|
+
context 'BADREQ/NOSRV log line' do
|
60
135
|
|
61
|
-
let(:
|
62
|
-
|
136
|
+
let(:message) do
|
137
|
+
'Jul 18 17:05:30 localhost haproxy[8247]: 188.223.50.7:51940 [18/Jul/2011:17:05:24.339] http_proxy_ads http_proxy_ads/<NOSRV> -1/-1/-1/-1/6001 408 212 - - cR-- 100/89/0/0/0 0/0 "<BADREQ>"'
|
138
|
+
end
|
63
139
|
|
64
|
-
it
|
65
|
-
|
66
|
-
|
67
|
-
it { should include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss") }
|
68
|
-
it { should_not have_key("http_version") }
|
140
|
+
it "matches" do
|
141
|
+
if ecs_compatibility?
|
142
|
+
expect(subject).to include("timestamp"=>"Jul 18 17:05:30")
|
69
143
|
|
70
|
-
|
71
|
-
|
144
|
+
expect(subject).to include("haproxy" => hash_including("frontend_name"=>"http_proxy_ads"))
|
145
|
+
expect(subject).to include("haproxy" => hash_including("backend_name"=>"http_proxy_ads"))
|
146
|
+
expect(subject['haproxy'].keys).to_not include('server_name')
|
147
|
+
expect(subject).to include("http"=>{"response"=>{"status_code"=>408}})
|
148
|
+
expect(subject['haproxy'].keys).to_not include("total_waiting_time_ms", "connection_wait_time_ms")
|
149
|
+
expect(subject).to include("haproxy" => hash_including("total_time_ms"=>"6001"))
|
150
|
+
expect(subject).to include("source" => hash_including("bytes"=>212))
|
151
|
+
expect(subject).to include("haproxy" => hash_including("termination_state"=>"cR--"))
|
152
|
+
expect(subject.keys).to_not include("url")
|
153
|
+
else
|
154
|
+
expect(subject).to include("backend_name"=>"http_proxy_ads", "frontend_name"=>"http_proxy_ads", "server_name"=>"<NOSRV>")
|
155
|
+
expect(subject).to include("http_status_code"=>"408")
|
156
|
+
expect(subject).to include("time_backend_connect"=>"-1", "time_queue"=>"-1", "time_backend_response"=>"-1")
|
157
|
+
expect(subject).to include("captured_request_cookie"=>"-", "captured_response_cookie"=>"-")
|
158
|
+
expect(subject).to include("bytes_read"=>"212")
|
159
|
+
expect(subject).to include("termination_state"=>"cR--")
|
160
|
+
end
|
72
161
|
end
|
73
162
|
|
74
163
|
end
|
75
164
|
|
76
165
|
end
|
166
|
+
|
167
|
+
describe_pattern "HAPROXYHTTPBASE", ['ecs-v1', 'legacy'] do
|
168
|
+
|
169
|
+
context "log line without syslog specific entries" do # This mimics an event coming from a syslog input.
|
170
|
+
|
171
|
+
let(:message) do
|
172
|
+
'127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/+99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET / HTTP/1.1"'
|
173
|
+
end
|
174
|
+
|
175
|
+
it 'matches' do
|
176
|
+
if ecs_compatibility?
|
177
|
+
expect(subject).to include("source"=>{"port"=>39759, "address"=>"127.0.0.1", "bytes"=>83285})
|
178
|
+
expect(subject).to include("haproxy"=>hash_including("server_queue"=>0,
|
179
|
+
"http"=>{
|
180
|
+
"request"=>{"time_wait_ms"=>0, "captured_headers"=>"77.24.148.74", "time_wait_without_data_ms"=>48082}
|
181
|
+
},
|
182
|
+
|
183
|
+
# NOTE: this is why we do not type-cast to :int
|
184
|
+
# a '+' sign is prepended before the value, indicating that the final one will be larger
|
185
|
+
"total_time_ms" => "+99627"
|
186
|
+
))
|
187
|
+
expect(subject).to include("url"=>{"path"=>"/", "original"=>"/"})
|
188
|
+
else
|
189
|
+
# Assume 'program' would be matched by the syslog input.
|
190
|
+
expect(subject).to include("client_ip" => "127.0.0.1")
|
191
|
+
expect(subject).to include("server_name" => "instance8")
|
192
|
+
expect(subject).to include("http_verb" => "GET", "http_request"=>"/", "http_version" => '1.1')
|
193
|
+
expect(subject).to include("time_duration" => "+99627")
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
end
|
198
|
+
|
199
|
+
context "(incomplete) log line that is truncated and thus not ending with a double quote or HTTP version" do
|
200
|
+
|
201
|
+
let(:message) do
|
202
|
+
'Jul 31 22:20:22 loadbalancer haproxy[1190]: 203.0.113.54:59968 [31/Jul/2017:22:20:22.447] loadbalancer default/instance8 135/0/1/19/156 200 1015 - - --VR 8/8/0/0/0 0/0 "GET /path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss'
|
203
|
+
end
|
204
|
+
|
205
|
+
it 'matches' do
|
206
|
+
if ecs_compatibility?
|
207
|
+
# due compatibility with the legacy pattern we match the incomplete "REQUEST LINE ... (wout the ending '"')
|
208
|
+
expect(subject).to include("http"=>{"response"=>{"status_code"=>200}, "request"=>{"method"=>"GET"}})
|
209
|
+
expect(subject).to include("url"=>hash_including("original"=>"/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss"))
|
210
|
+
else
|
211
|
+
expect(subject).to include("client_ip" => "203.0.113.54")
|
212
|
+
expect(subject).to include("http_verb" => "GET")
|
213
|
+
expect(subject).to include("server_name" => "instance8")
|
214
|
+
expect(subject).to include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss")
|
215
|
+
expect(subject).to_not have_key("http_version")
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
end
|
220
|
+
|
221
|
+
|
222
|
+
context "connect line with host:port url" do
|
223
|
+
|
224
|
+
let(:message) do
|
225
|
+
'Nov 4 08:32:18 debian10 haproxy[3666]: 127.0.0.1:34500 [04/Nov/2020:08:32:18.194] samplefrontend backendnodes/node1 0/0/0/0/0 405 501 - - ---- 1/1/0/1/0 0/0 "CONNECT localhost:8080 HTTP/1.1"'
|
226
|
+
end
|
227
|
+
|
228
|
+
it 'matches' do
|
229
|
+
if ecs_compatibility?
|
230
|
+
expect(subject).to include("http"=>hash_including("request"=>{"method"=>"CONNECT"}))
|
231
|
+
expect(subject).to include("url"=>{"port"=>8080, "original"=>"localhost:8080", "domain"=>"localhost"})
|
232
|
+
else
|
233
|
+
expect(subject).to include("http_verb" => "CONNECT")
|
234
|
+
expect(subject).to include("http_host" => "localhost:8080")
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
end
|
239
|
+
|
240
|
+
end
|
241
|
+
|
242
|
+
describe_pattern "HAPROXYTCP", ['legacy', 'ecs-v1'] do
|
243
|
+
|
244
|
+
let(:message) do
|
245
|
+
'Sep 20 15:44:23 127.0.0.1 haproxy[25457]: 127.0.0.1:40962 [20/Sep/2018:15:44:23.285] main app/<NOSRV> -1/-1/1 212 SC 1/1/0/0/0 0/0'
|
246
|
+
end
|
247
|
+
|
248
|
+
it 'matches' do
|
249
|
+
if ecs_compatibility?
|
250
|
+
expect(subject).to include(
|
251
|
+
"timestamp"=>"Sep 20 15:44:23",
|
252
|
+
"host"=>{"hostname"=>"127.0.0.1"},
|
253
|
+
"process"=>{"pid"=>25457, "name"=>"haproxy"},
|
254
|
+
"source"=>{"port"=>40962, "address"=>"127.0.0.1", "bytes"=>212},
|
255
|
+
"haproxy"=>{
|
256
|
+
"request_date"=>"20/Sep/2018:15:44:23.285",
|
257
|
+
"frontend_name"=>"main", "backend_name"=>"app",
|
258
|
+
"total_time_ms"=>"1",
|
259
|
+
"termination_state"=>"SC",
|
260
|
+
"connections"=>{"active"=>1, "backend"=>0, "retries"=>0, "server"=>0, "frontend"=>1},
|
261
|
+
"server_queue"=>0, "backend_queue"=>0
|
262
|
+
})
|
263
|
+
else
|
264
|
+
expect(subject).to include(
|
265
|
+
"syslog_timestamp"=>"Sep 20 15:44:23",
|
266
|
+
"syslog_server"=>"127.0.0.1",
|
267
|
+
"program"=>"haproxy", "pid"=>"25457",
|
268
|
+
"client_ip"=>"127.0.0.1", "client_port"=>"40962",
|
269
|
+
"accept_date"=>"20/Sep/2018:15:44:23.285",
|
270
|
+
"frontend_name"=>"main",
|
271
|
+
"backend_name"=>"app",
|
272
|
+
"server_name"=>"<NOSRV>",
|
273
|
+
"time_backend_connect"=>"-1",
|
274
|
+
"time_queue"=>"-1",
|
275
|
+
"time_duration"=>"1",
|
276
|
+
"bytes_read"=>"212",
|
277
|
+
"termination_state"=>"SC",
|
278
|
+
"actconn"=>"1", "feconn"=>"1", "beconn"=>"0", "backend_queue"=>"0", "retries"=>"0",
|
279
|
+
"srv_queue"=>"0", "srvconn"=>"0",
|
280
|
+
)
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
end
|
data/spec/patterns/httpd_spec.rb
CHANGED
@@ -2,30 +2,51 @@
|
|
2
2
|
require "spec_helper"
|
3
3
|
require "logstash/patterns/core"
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
let(:pattern) { 'HTTPD_COMBINEDLOG' }
|
8
|
-
let(:grok) { grok_match(pattern, message) }
|
5
|
+
describe_pattern "HTTPD_COMBINEDLOG", ['legacy', 'ecs-v1'] do
|
9
6
|
|
10
7
|
context "typical test case" do
|
11
8
|
|
12
9
|
let(:message) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'}
|
13
10
|
|
14
11
|
it "matches" do
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
12
|
+
if ecs_compatibility?
|
13
|
+
expect(grok).to include(
|
14
|
+
"http" => {
|
15
|
+
"request" => {
|
16
|
+
"method" => "GET",
|
17
|
+
"referrer" => "http://semicomplete.com/presentations/logstash-monitorama-2013/"
|
18
|
+
},
|
19
|
+
"response" => {
|
20
|
+
"body" => { "bytes" => 203023 },
|
21
|
+
"status_code" => 200
|
22
|
+
},
|
23
|
+
"version"=>"1.1"
|
24
|
+
},
|
25
|
+
"source" => { "address" => "83.149.9.216" },
|
26
|
+
"url" => { "original" => "/presentations/logstash-monitorama-2013/images/kibana-search.png" },
|
27
|
+
"user_agent" => { "original" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" }
|
28
|
+
)
|
29
|
+
else
|
30
|
+
expect(grok).to include(
|
31
|
+
'clientip' => '83.149.9.216',
|
32
|
+
'verb' => 'GET',
|
33
|
+
'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
|
34
|
+
'httpversion' => '1.1',
|
35
|
+
'response' => '200',
|
36
|
+
'bytes' => '203023',
|
37
|
+
'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
|
38
|
+
'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
|
39
|
+
)
|
40
|
+
end
|
25
41
|
end
|
26
42
|
|
27
43
|
it "does not capture 'null' fields" do
|
28
|
-
|
44
|
+
if ecs_compatibility?
|
45
|
+
expect(grok.keys).to_not include('user') # 'user' => 'name'
|
46
|
+
expect(grok.keys).to_not include('apache') # apache.access.user.identity
|
47
|
+
else
|
48
|
+
expect(grok).to include('auth' => '-', 'ident' => '-')
|
49
|
+
end
|
29
50
|
end
|
30
51
|
|
31
52
|
end
|
@@ -35,7 +56,11 @@ describe "HTTPD_COMBINEDLOG" do
|
|
35
56
|
let(:message) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'}
|
36
57
|
|
37
58
|
it "gets captured" do
|
38
|
-
|
59
|
+
if ecs_compatibility?
|
60
|
+
expect(grok).to include("user" => { 'name' => "username@example.com" })
|
61
|
+
else
|
62
|
+
expect(grok).to include("auth" => "username@example.com")
|
63
|
+
end
|
39
64
|
end
|
40
65
|
|
41
66
|
end
|
@@ -45,121 +70,212 @@ describe "HTTPD_COMBINEDLOG" do
|
|
45
70
|
let(:message) { '83.149.9.216 - a.user [11/Jan/2020:23:05:27 +0100] "OPTIONS /remote.php/ HTTP/1.1" - 7908 "-" "monitoring-client (v2.2)"' }
|
46
71
|
|
47
72
|
it 'matches' do
|
48
|
-
|
73
|
+
if ecs_compatibility?
|
74
|
+
expect(grok).to include("http" => hash_including("response" => hash_including("body" => { "bytes" => 7908 })))
|
75
|
+
expect(grok).to include("http" => hash_including("request" => { "method" => "OPTIONS" }, "version" => "1.1"))
|
76
|
+
expect(grok).to include(
|
77
|
+
"url" => { "original" => "/remote.php/" },
|
78
|
+
"user_agent" => { "original" => "monitoring-client (v2.2)" }
|
79
|
+
)
|
80
|
+
else
|
81
|
+
expect(grok).to include("verb" => "OPTIONS", 'request' => '/remote.php/', 'httpversion' => '1.1', "bytes" => '7908')
|
82
|
+
end
|
49
83
|
end
|
50
84
|
|
51
85
|
it 'does not capture optional response code' do
|
52
|
-
|
86
|
+
if ecs_compatibility?
|
87
|
+
expect(grok['http']['response'].keys).to_not include("status_code")
|
88
|
+
else
|
89
|
+
expect(grok.keys).to_not include("response")
|
90
|
+
end
|
53
91
|
end
|
54
92
|
|
55
93
|
end
|
56
94
|
|
57
95
|
end
|
58
96
|
|
59
|
-
|
60
|
-
|
61
|
-
let(:pattern) { 'HTTPD_ERRORLOG' }
|
62
|
-
let(:grok) { grok_match(pattern, message) }
|
97
|
+
describe_pattern "HTTPD_ERRORLOG", ['legacy', 'ecs-v1'] do
|
63
98
|
|
64
|
-
context "
|
99
|
+
context "a full httpd 2.4 message" do
|
65
100
|
let(:message) do
|
66
101
|
"[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product"
|
67
102
|
end
|
68
|
-
it "generates the fields" do
|
69
103
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
104
|
+
it "generates the fields" do
|
105
|
+
expect(grok).to include('timestamp' => 'Mon Aug 31 09:30:48.958285 2015')
|
106
|
+
if ecs_compatibility?
|
107
|
+
expect(grok).to include("log" => { "level" => "error" })
|
108
|
+
expect(grok).to include("process" => { "pid" => 28787, "thread" => { "id" => 140169587934976 } })
|
109
|
+
expect(grok).to include("source" => { "address"=>"58.13.45.166", "port" => 59307 })
|
110
|
+
expect(grok).to include("error" => { "code" => 'AH01075' })
|
111
|
+
expect(grok).to include("apache" => { "error" => {
|
112
|
+
"module" => "proxy_fcgi",
|
113
|
+
"proxy" => { "error" => { "code" => '70008', "message" => "Partial results are valid but processing is incomplete" }}}
|
114
|
+
})
|
115
|
+
else
|
116
|
+
expect(grok).to include(
|
117
|
+
'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
|
118
|
+
'module' => 'proxy_fcgi',
|
119
|
+
'loglevel' => 'error',
|
120
|
+
'pid' => '28787',
|
121
|
+
'tid' => '140169587934976',
|
122
|
+
'proxy_errorcode' => '70008',
|
123
|
+
'proxy_message' => 'Partial results are valid but processing is incomplete',
|
124
|
+
'clientip' => '58.13.45.166',
|
125
|
+
'clientport' => '59307',
|
126
|
+
'errorcode' => 'AH01075'
|
127
|
+
)
|
128
|
+
end
|
129
|
+
expect(grok).to include('message' => [ message, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ])
|
83
130
|
end
|
84
131
|
end
|
85
132
|
|
86
|
-
context "
|
133
|
+
context "a httpd 2.2 log message" do
|
87
134
|
let(:message) do
|
88
135
|
"[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com"
|
89
136
|
end
|
137
|
+
|
90
138
|
it "generates the fields" do
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
139
|
+
if ecs_compatibility?
|
140
|
+
expect(grok).to include(
|
141
|
+
"timestamp"=>"Mon Aug 31 16:27:04 2015",
|
142
|
+
"log"=>{"level"=>"error"},
|
143
|
+
"source"=>{"address"=>"10.17.42.3"})
|
144
|
+
expect(grok.keys).to_not include("error") # error.code
|
145
|
+
else
|
146
|
+
expect(grok).to include(
|
147
|
+
'timestamp' => 'Mon Aug 31 16:27:04 2015',
|
148
|
+
'loglevel' => 'error',
|
149
|
+
'clientip' => '10.17.42.3'
|
150
|
+
)
|
151
|
+
expect(grok.keys).to_not include('errorcode')
|
152
|
+
end
|
153
|
+
expect(grok).to include('message' => [ message, 'Premature end of script headers: example.com' ])
|
97
154
|
end
|
98
155
|
end
|
99
156
|
|
100
|
-
context "
|
157
|
+
context "a short httpd 2.4 message" do
|
101
158
|
let(:value1) {
|
102
159
|
"[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'"
|
103
160
|
}
|
104
161
|
it "generates the fields" do
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
162
|
+
match_result = grok_match(pattern, value1)
|
163
|
+
expect(match_result).to include('timestamp' => 'Mon Aug 31 07:15:38.664897 2015')
|
164
|
+
if ecs_compatibility?
|
165
|
+
expect(match_result).to include(
|
166
|
+
"apache"=>{"error"=>{"module"=>"proxy_fcgi"}},
|
167
|
+
"log"=>{"level"=>"error"},
|
168
|
+
"process"=>{"pid"=>28786, "thread"=>{"id"=>140169629898496}},
|
169
|
+
"source"=>{"address"=>"81.139.1.34", "port"=>52042},
|
170
|
+
"error"=>{"code"=>"AH01071"},
|
171
|
+
)
|
172
|
+
else
|
173
|
+
expect(match_result).to include(
|
174
|
+
'module' => 'proxy_fcgi',
|
175
|
+
'loglevel' => 'error',
|
176
|
+
'pid' => '28786',
|
177
|
+
'tid' => '140169629898496',
|
178
|
+
'clientip' => '81.139.1.34',
|
179
|
+
'clientport' => '52042',
|
180
|
+
'errorcode' => 'AH01071'
|
181
|
+
)
|
182
|
+
end
|
183
|
+
expect(match_result).to include('message' => [ value1, "Got error 'Primary script unknown\n'" ])
|
116
184
|
end
|
117
185
|
|
118
186
|
let(:value2) {
|
119
187
|
"[Thu Apr 27 10:39:46.719636 2017] [php7:notice] [pid 17] [client 10.255.0.3:49580] Test error log record"
|
120
188
|
}
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
189
|
+
it "generates the fields" do
|
190
|
+
match_result = grok_match(pattern, value2)
|
191
|
+
expect(match_result).to include('timestamp' => 'Thu Apr 27 10:39:46.719636 2017')
|
192
|
+
if ecs_compatibility?
|
193
|
+
expect(match_result).to include(
|
194
|
+
"apache"=>{"error"=>{"module"=>"php7"}},
|
195
|
+
"log"=>{"level"=>"notice"},
|
196
|
+
"process"=>{"pid"=>17},
|
197
|
+
"source"=>{"port"=>49580, "address"=>"10.255.0.3"}
|
198
|
+
)
|
199
|
+
else
|
200
|
+
expect(match_result).to include(
|
201
|
+
'module' => 'php7',
|
202
|
+
'loglevel' => 'notice',
|
203
|
+
'pid' => '17',
|
204
|
+
'clientip' => '10.255.0.3',
|
205
|
+
'clientport' => '49580'
|
206
|
+
)
|
207
|
+
end
|
208
|
+
expect(match_result).to include('message' => [ value2, "Test error log record" ])
|
131
209
|
end
|
132
210
|
end
|
133
211
|
|
134
|
-
context "
|
212
|
+
context "a httpd 2.4 restart message" do
|
135
213
|
let(:value1) {
|
136
214
|
"[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations"
|
137
215
|
}
|
138
216
|
it "generates the fields" do
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
217
|
+
match_result = grok_match(pattern, value1)
|
218
|
+
expect(match_result).to include('timestamp' => 'Mon Aug 31 06:29:47.406518 2015')
|
219
|
+
if ecs_compatibility?
|
220
|
+
expect(match_result).to include(
|
221
|
+
"apache"=>{"error"=>{"module"=>"mpm_event"}},
|
222
|
+
"log"=>{"level"=>"notice"},
|
223
|
+
"process"=>{"pid"=>24968, "thread"=>{"id"=>140169861986176}},
|
224
|
+
"error"=>{"code"=>"AH00489"}
|
225
|
+
)
|
226
|
+
|
227
|
+
else
|
228
|
+
expect(match_result).to include(
|
229
|
+
'module' => 'mpm_event',
|
230
|
+
'loglevel' => 'notice',
|
231
|
+
'pid' => '24968',
|
232
|
+
'tid' => '140169861986176',
|
233
|
+
'errorcode' => 'AH00489'
|
234
|
+
)
|
235
|
+
end
|
236
|
+
expect(match_result).to include('message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ])
|
148
237
|
end
|
149
238
|
|
150
239
|
let(:value2) {
|
151
240
|
"[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'"
|
152
241
|
}
|
153
242
|
it "generates the fields" do
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
243
|
+
match_result = grok_match(pattern, value2)
|
244
|
+
expect(match_result).to include('timestamp' => 'Mon Aug 31 06:29:47.406530 2015')
|
245
|
+
if ecs_compatibility?
|
246
|
+
expect(match_result).to include(
|
247
|
+
"apache"=>{"error"=>{"module"=>"core"}},
|
248
|
+
"log"=>{"level"=>"notice"},
|
249
|
+
"process"=>{"pid"=>24968, "thread"=>{"id"=>140169861986176}},
|
250
|
+
"error"=>{"code"=>"AH00094"}
|
251
|
+
)
|
252
|
+
else
|
253
|
+
expect(match_result).to include(
|
254
|
+
'module' => 'core',
|
255
|
+
'loglevel' => 'notice',
|
256
|
+
'pid' => '24968',
|
257
|
+
'tid' => '140169861986176',
|
258
|
+
'errorcode' => 'AH00094'
|
259
|
+
)
|
260
|
+
end
|
261
|
+
expect(match_result).to include('message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ])
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
context "a httpd 2.4 message witout module" do
|
266
|
+
let(:message) do
|
267
|
+
"[Tue Apr 14 14:27:52.605084 2020] [:error] [pid 5688] [client 192.168.10.110:8196] script '/login/wp-login.php' not found or unable to stat"
|
268
|
+
end
|
269
|
+
|
270
|
+
it "matches" do
|
271
|
+
expect(grok).to include('timestamp' => 'Tue Apr 14 14:27:52.605084 2020')
|
272
|
+
if ecs_compatibility?
|
273
|
+
expect(grok).to include("log"=>{"level" => "error"})
|
274
|
+
expect(grok).to include("process"=>{"pid" => 5688})
|
275
|
+
expect( ((grok['apache'] || {})['error'] || {}).keys ).to_not include('module')
|
276
|
+
else
|
277
|
+
expect(grok).to include('loglevel' => 'error', 'pid' => '5688')
|
278
|
+
end
|
163
279
|
end
|
164
280
|
end
|
165
281
|
|
@@ -169,15 +285,20 @@ describe "HTTPD_ERRORLOG" do
|
|
169
285
|
end
|
170
286
|
|
171
287
|
it 'matches imperfectly (legacy)' do
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
288
|
+
if ecs_compatibility?
|
289
|
+
pending
|
290
|
+
raise NotImplementedError, "TODO: would be nice to 'improve' matching on these debug logs as well"
|
291
|
+
else
|
292
|
+
expect(grok).to include({
|
293
|
+
"timestamp"=>"Fri Feb 01 22:03:08.319124 2019",
|
294
|
+
"module"=>"authz_core",
|
295
|
+
"loglevel"=>"debug",
|
296
|
+
"pid"=>"9",
|
297
|
+
"tid"=>"140597881775872",
|
298
|
+
"errorcode"=>"mod_authz_core.c(820)",
|
299
|
+
"message"=>[message, "[client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted"]
|
300
|
+
})
|
301
|
+
end
|
181
302
|
end
|
182
303
|
end
|
183
304
|
|