logstash-patterns-core 4.2.0 → 4.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +98 -0
- data/Gemfile +3 -0
- data/README.md +11 -18
- data/lib/logstash/patterns/core.rb +11 -3
- data/logstash-patterns-core.gemspec +1 -1
- data/patterns/ecs-v1/aws +28 -0
- data/patterns/ecs-v1/bacula +53 -0
- data/patterns/ecs-v1/bind +13 -0
- data/patterns/ecs-v1/bro +30 -0
- data/patterns/ecs-v1/exim +26 -0
- data/patterns/ecs-v1/firewalls +111 -0
- data/patterns/ecs-v1/grok-patterns +95 -0
- data/patterns/ecs-v1/haproxy +40 -0
- data/patterns/ecs-v1/httpd +17 -0
- data/patterns/ecs-v1/java +34 -0
- data/patterns/ecs-v1/junos +13 -0
- data/patterns/ecs-v1/linux-syslog +16 -0
- data/patterns/{maven → ecs-v1/maven} +0 -0
- data/patterns/ecs-v1/mcollective +4 -0
- data/patterns/ecs-v1/mongodb +7 -0
- data/patterns/ecs-v1/nagios +124 -0
- data/patterns/ecs-v1/postgresql +2 -0
- data/patterns/ecs-v1/rails +13 -0
- data/patterns/ecs-v1/redis +3 -0
- data/patterns/ecs-v1/ruby +2 -0
- data/patterns/ecs-v1/squid +6 -0
- data/patterns/ecs-v1/zeek +33 -0
- data/patterns/{aws → legacy/aws} +1 -1
- data/patterns/{bacula → legacy/bacula} +5 -5
- data/patterns/legacy/bind +3 -0
- data/patterns/{bro → legacy/bro} +0 -0
- data/patterns/{exim → legacy/exim} +8 -2
- data/patterns/{firewalls → legacy/firewalls} +2 -2
- data/patterns/{grok-patterns → legacy/grok-patterns} +0 -0
- data/patterns/{haproxy → legacy/haproxy} +0 -0
- data/patterns/{httpd → legacy/httpd} +1 -1
- data/patterns/{java → legacy/java} +0 -0
- data/patterns/{junos → legacy/junos} +0 -0
- data/patterns/{linux-syslog → legacy/linux-syslog} +0 -0
- data/patterns/legacy/maven +1 -0
- data/patterns/{mcollective → legacy/mcollective} +0 -0
- data/patterns/{mcollective-patterns → legacy/mcollective-patterns} +0 -0
- data/patterns/{mongodb → legacy/mongodb} +0 -0
- data/patterns/{nagios → legacy/nagios} +0 -0
- data/patterns/{postgresql → legacy/postgresql} +0 -0
- data/patterns/{rails → legacy/rails} +0 -0
- data/patterns/{redis → legacy/redis} +0 -0
- data/patterns/{ruby → legacy/ruby} +0 -0
- data/patterns/legacy/squid +4 -0
- data/spec/patterns/aws_spec.rb +395 -0
- data/spec/patterns/bacula_spec.rb +367 -0
- data/spec/patterns/bind_spec.rb +78 -0
- data/spec/patterns/bro_spec.rb +613 -0
- data/spec/patterns/core_spec.rb +51 -9
- data/spec/patterns/exim_spec.rb +201 -0
- data/spec/patterns/firewalls_spec.rb +669 -66
- data/spec/patterns/haproxy_spec.rb +246 -38
- data/spec/patterns/httpd_spec.rb +215 -94
- data/spec/patterns/java_spec.rb +357 -27
- data/spec/patterns/junos_spec.rb +101 -0
- data/spec/patterns/mcollective_spec.rb +35 -0
- data/spec/patterns/mongodb_spec.rb +170 -33
- data/spec/patterns/nagios_spec.rb +296 -79
- data/spec/patterns/netscreen_spec.rb +123 -0
- data/spec/patterns/rails3_spec.rb +87 -29
- data/spec/patterns/redis_spec.rb +157 -121
- data/spec/patterns/shorewall_spec.rb +85 -74
- data/spec/patterns/squid_spec.rb +139 -0
- data/spec/patterns/syslog_spec.rb +266 -22
- data/spec/spec_helper.rb +80 -6
- metadata +64 -28
- data/patterns/bind +0 -3
- data/patterns/squid +0 -4
- data/spec/patterns/bro.rb +0 -126
- data/spec/patterns/s3_spec.rb +0 -173
@@ -2,19 +2,60 @@
|
|
2
2
|
require "spec_helper"
|
3
3
|
require "logstash/patterns/core"
|
4
4
|
|
5
|
-
|
5
|
+
describe_pattern "HAPROXYHTTP", ['legacy', 'ecs-v1'] do
|
6
6
|
|
7
|
-
|
7
|
+
context "log line from raw syslog line" do
|
8
8
|
|
9
|
-
|
9
|
+
let(:message) do
|
10
|
+
'Dec 9 13:01:26 localhost haproxy[28029]: 127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"'
|
11
|
+
end
|
12
|
+
|
13
|
+
it "matches" do
|
14
|
+
if ecs_compatibility?
|
15
|
+
expect(subject).to include("timestamp"=>"Dec 9 13:01:26")
|
16
|
+
expect(subject).to include("host"=>{"hostname"=>"localhost"})
|
17
|
+
expect(subject).to include("process"=>{"pid"=>28029, "name"=>"haproxy"})
|
18
|
+
expect(subject).to include("source"=>{"port"=>39759, "address"=>"127.0.0.1", "bytes"=>83285})
|
19
|
+
expect(subject).to include("haproxy" => hash_including("request_date"=>"09/Dec/2013:12:59:46.633"))
|
20
|
+
expect(subject).to include("haproxy" => hash_including("frontend_name"=>"loadbalancer", "backend_name"=>"default", "server_name"=>"instance8"))
|
21
|
+
expect(subject).to include("haproxy" => hash_including(
|
22
|
+
"total_waiting_time_ms"=>51536, "connection_wait_time_ms"=>1, "total_time_ms"=>"99627",
|
23
|
+
"http" => hash_including("request"=>hash_including("time_wait_ms"=>0, "time_wait_without_data_ms"=>48082))
|
24
|
+
))
|
25
|
+
expect(subject).to include("http" => hash_including("response"=>{"status_code"=>200}))
|
26
|
+
|
27
|
+
expect(subject).to include("haproxy" => hash_including("termination_state"=>"----"))
|
28
|
+
|
29
|
+
expect(subject).to include("haproxy" => hash_including("connections"=>{"active"=>87, "frontend"=>87, "backend"=>87, "server"=>1, "retries"=>0}))
|
30
|
+
expect(subject).to include("haproxy" => hash_including("backend_queue"=>67, "server_queue"=>0))
|
10
31
|
|
11
|
-
|
12
|
-
subject { grok_match(haproxyhttp_pattern, value) }
|
32
|
+
expect(subject).to include("http" => hash_including("request" => {"method"=>'GET'}, "version" => '1.1'))
|
13
33
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
34
|
+
expect(subject).to include("url" => { "original"=>"/path/to/image", "path"=>"/path/to/image" })
|
35
|
+
else
|
36
|
+
expect(subject).to include("syslog_timestamp" => "Dec 9 13:01:26")
|
37
|
+
expect(subject).to include("syslog_server" => "localhost")
|
38
|
+
expect(subject).to include("http_request" => "/path/to/image", "http_status_code" => "200", "http_verb" => "GET", "http_version" => "1.1")
|
39
|
+
expect(subject).to include("program" => "haproxy")
|
40
|
+
expect(subject).to include("client_ip" => "127.0.0.1")
|
41
|
+
expect(subject).to include("http_verb" => "GET")
|
42
|
+
expect(subject).to include("server_name" => "instance8")
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
it "has no captured cookies" do
|
47
|
+
if ecs_compatibility?
|
48
|
+
expect((subject['haproxy']['http']['request'] || {}).keys).to_not include('captured_cookie')
|
49
|
+
expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_cookie')
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
it "includes header captures" do
|
54
|
+
if ecs_compatibility?
|
55
|
+
expect((subject['haproxy']['http'])).to include('request' => hash_including('captured_headers' => '77.24.148.74'))
|
56
|
+
expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_headers')
|
57
|
+
end
|
58
|
+
end
|
18
59
|
|
19
60
|
it "generates a message field" do
|
20
61
|
expect(subject["message"]).to include("loadbalancer default/instance8")
|
@@ -22,55 +63,222 @@ describe "HAPROXY" do
|
|
22
63
|
|
23
64
|
end
|
24
65
|
|
25
|
-
context "
|
66
|
+
context "log line (without headers) from raw syslog line with ISO8601 timestamp" do
|
26
67
|
|
27
|
-
let(:
|
28
|
-
|
68
|
+
let(:message) do
|
69
|
+
'2015-08-26T02:09:48+02:00 localhost haproxy[14389]: 5.196.2.38:39527 [03/Nov/2015:06:25:25.105] services~ def/api 4599/0/0/428/5027 304 320 - - ---- 1/1/0/1/0 0/0 "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1"'
|
70
|
+
end
|
29
71
|
|
30
|
-
it
|
31
|
-
|
32
|
-
|
33
|
-
|
72
|
+
it "matches" do
|
73
|
+
if ecs_compatibility?
|
74
|
+
expect(subject).to include("timestamp"=>"2015-08-26T02:09:48+02:00")
|
75
|
+
expect(subject).to include("host"=>{"hostname"=>"localhost"})
|
76
|
+
expect(subject).to include("process"=>{"pid"=>14389, "name"=>"haproxy"})
|
34
77
|
|
35
|
-
|
36
|
-
|
78
|
+
expect(subject).to include("haproxy" => hash_including("connections"=>{"active"=>1, "frontend"=>1, "backend"=>0, "server"=>1, "retries"=>0}))
|
79
|
+
expect(subject).to include("haproxy" => hash_including("backend_queue"=>0, "server_queue"=>0))
|
80
|
+
|
81
|
+
expect(subject).to include("haproxy" => hash_including("frontend_name"=>"services~"))
|
82
|
+
|
83
|
+
expect(subject).to include("http"=>{"response"=>{"status_code"=>304}, "version"=>"1.1", "request"=>{"method"=>"GET"}})
|
84
|
+
expect(subject).to include("url"=>hash_including("path"=>"/component---src-pages-index-js-4b15624544f97cf0bb8f.js"))
|
85
|
+
else
|
86
|
+
expect(subject).to include("program" => "haproxy")
|
87
|
+
expect(subject).to include("client_ip" => "5.196.2.38")
|
88
|
+
expect(subject).to include("http_verb" => "GET")
|
89
|
+
expect(subject).to include("server_name" => "api")
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
it "has no header captures" do
|
94
|
+
if ecs_compatibility?
|
95
|
+
expect((subject['haproxy']['http']['request'] || {}).keys).to_not include('captured_headers')
|
96
|
+
expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_headers')
|
97
|
+
end
|
37
98
|
end
|
38
99
|
|
39
100
|
end
|
40
101
|
|
41
|
-
|
102
|
+
context 'log line with both request/response headers' do
|
42
103
|
|
43
|
-
|
104
|
+
let(:message) do
|
105
|
+
'Jul 30 09:03:52 home.host haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} "GET http://192.168.0.12:8080/serv/login.php?lang=en&profile=2 HTTP/1.1"'
|
106
|
+
end
|
44
107
|
|
45
|
-
|
46
|
-
|
108
|
+
it "matches" do
|
109
|
+
if ecs_compatibility?
|
110
|
+
expect(subject).to include("timestamp"=>"Jul 30 09:03:52")
|
111
|
+
expect(subject).to include("host"=>{"hostname"=>"home.host"})
|
47
112
|
|
48
|
-
|
49
|
-
it { should include("client_ip" => "127.0.0.1") }
|
50
|
-
it { should include("http_verb" => "GET") }
|
51
|
-
it { should include("server_name" => "instance8") }
|
113
|
+
expect(subject).to include("haproxy" => hash_including("frontend_name"=>"incoming~"))
|
52
114
|
|
53
|
-
|
54
|
-
|
115
|
+
expect(subject).to include("http"=>{"response"=>{"status_code"=>304}, "version"=>"1.1", "request"=>{"method"=>"GET"}})
|
116
|
+
expect(subject).to include("url"=>hash_including("scheme"=>"http", "domain"=>"192.168.0.12", "port"=>8080,
|
117
|
+
"path"=>"/serv/login.php", "query"=>"lang=en&profile=2",
|
118
|
+
"original"=>"http://192.168.0.12:8080/serv/login.php?lang=en&profile=2"))
|
119
|
+
else
|
120
|
+
expect(subject).to include("client_ip" => "1.2.3.4")
|
121
|
+
expect(subject).to include("http_verb" => "GET")
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
it "has header captures" do
|
126
|
+
if ecs_compatibility?
|
127
|
+
expect((subject['haproxy']['http']['request'])).to include('captured_headers' => 'docs.example.internal||')
|
128
|
+
expect((subject['haproxy']['http']['response'])).to include('captured_headers' => '|||')
|
129
|
+
end
|
55
130
|
end
|
56
131
|
|
57
132
|
end
|
58
133
|
|
59
|
-
context
|
134
|
+
context 'BADREQ/NOSRV log line' do
|
60
135
|
|
61
|
-
let(:
|
62
|
-
|
136
|
+
let(:message) do
|
137
|
+
'Jul 18 17:05:30 localhost haproxy[8247]: 188.223.50.7:51940 [18/Jul/2011:17:05:24.339] http_proxy_ads http_proxy_ads/<NOSRV> -1/-1/-1/-1/6001 408 212 - - cR-- 100/89/0/0/0 0/0 "<BADREQ>"'
|
138
|
+
end
|
63
139
|
|
64
|
-
it
|
65
|
-
|
66
|
-
|
67
|
-
it { should include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss") }
|
68
|
-
it { should_not have_key("http_version") }
|
140
|
+
it "matches" do
|
141
|
+
if ecs_compatibility?
|
142
|
+
expect(subject).to include("timestamp"=>"Jul 18 17:05:30")
|
69
143
|
|
70
|
-
|
71
|
-
|
144
|
+
expect(subject).to include("haproxy" => hash_including("frontend_name"=>"http_proxy_ads"))
|
145
|
+
expect(subject).to include("haproxy" => hash_including("backend_name"=>"http_proxy_ads"))
|
146
|
+
expect(subject['haproxy'].keys).to_not include('server_name')
|
147
|
+
expect(subject).to include("http"=>{"response"=>{"status_code"=>408}})
|
148
|
+
expect(subject['haproxy'].keys).to_not include("total_waiting_time_ms", "connection_wait_time_ms")
|
149
|
+
expect(subject).to include("haproxy" => hash_including("total_time_ms"=>"6001"))
|
150
|
+
expect(subject).to include("source" => hash_including("bytes"=>212))
|
151
|
+
expect(subject).to include("haproxy" => hash_including("termination_state"=>"cR--"))
|
152
|
+
expect(subject.keys).to_not include("url")
|
153
|
+
else
|
154
|
+
expect(subject).to include("backend_name"=>"http_proxy_ads", "frontend_name"=>"http_proxy_ads", "server_name"=>"<NOSRV>")
|
155
|
+
expect(subject).to include("http_status_code"=>"408")
|
156
|
+
expect(subject).to include("time_backend_connect"=>"-1", "time_queue"=>"-1", "time_backend_response"=>"-1")
|
157
|
+
expect(subject).to include("captured_request_cookie"=>"-", "captured_response_cookie"=>"-")
|
158
|
+
expect(subject).to include("bytes_read"=>"212")
|
159
|
+
expect(subject).to include("termination_state"=>"cR--")
|
160
|
+
end
|
72
161
|
end
|
73
162
|
|
74
163
|
end
|
75
164
|
|
76
165
|
end
|
166
|
+
|
167
|
+
describe_pattern "HAPROXYHTTPBASE", ['ecs-v1', 'legacy'] do
|
168
|
+
|
169
|
+
context "log line without syslog specific entries" do # This mimics an event coming from a syslog input.
|
170
|
+
|
171
|
+
let(:message) do
|
172
|
+
'127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/+99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET / HTTP/1.1"'
|
173
|
+
end
|
174
|
+
|
175
|
+
it 'matches' do
|
176
|
+
if ecs_compatibility?
|
177
|
+
expect(subject).to include("source"=>{"port"=>39759, "address"=>"127.0.0.1", "bytes"=>83285})
|
178
|
+
expect(subject).to include("haproxy"=>hash_including("server_queue"=>0,
|
179
|
+
"http"=>{
|
180
|
+
"request"=>{"time_wait_ms"=>0, "captured_headers"=>"77.24.148.74", "time_wait_without_data_ms"=>48082}
|
181
|
+
},
|
182
|
+
|
183
|
+
# NOTE: this is why we do not type-cast to :int
|
184
|
+
# a '+' sign is prepended before the value, indicating that the final one will be larger
|
185
|
+
"total_time_ms" => "+99627"
|
186
|
+
))
|
187
|
+
expect(subject).to include("url"=>{"path"=>"/", "original"=>"/"})
|
188
|
+
else
|
189
|
+
# Assume 'program' would be matched by the syslog input.
|
190
|
+
expect(subject).to include("client_ip" => "127.0.0.1")
|
191
|
+
expect(subject).to include("server_name" => "instance8")
|
192
|
+
expect(subject).to include("http_verb" => "GET", "http_request"=>"/", "http_version" => '1.1')
|
193
|
+
expect(subject).to include("time_duration" => "+99627")
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
end
|
198
|
+
|
199
|
+
context "(incomplete) log line that is truncated and thus not ending with a double quote or HTTP version" do
|
200
|
+
|
201
|
+
let(:message) do
|
202
|
+
'Jul 31 22:20:22 loadbalancer haproxy[1190]: 203.0.113.54:59968 [31/Jul/2017:22:20:22.447] loadbalancer default/instance8 135/0/1/19/156 200 1015 - - --VR 8/8/0/0/0 0/0 "GET /path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss'
|
203
|
+
end
|
204
|
+
|
205
|
+
it 'matches' do
|
206
|
+
if ecs_compatibility?
|
207
|
+
# due compatibility with the legacy pattern we match the incomplete "REQUEST LINE ... (wout the ending '"')
|
208
|
+
expect(subject).to include("http"=>{"response"=>{"status_code"=>200}, "request"=>{"method"=>"GET"}})
|
209
|
+
expect(subject).to include("url"=>hash_including("original"=>"/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss"))
|
210
|
+
else
|
211
|
+
expect(subject).to include("client_ip" => "203.0.113.54")
|
212
|
+
expect(subject).to include("http_verb" => "GET")
|
213
|
+
expect(subject).to include("server_name" => "instance8")
|
214
|
+
expect(subject).to include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss")
|
215
|
+
expect(subject).to_not have_key("http_version")
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
end
|
220
|
+
|
221
|
+
|
222
|
+
context "connect line with host:port url" do
|
223
|
+
|
224
|
+
let(:message) do
|
225
|
+
'Nov 4 08:32:18 debian10 haproxy[3666]: 127.0.0.1:34500 [04/Nov/2020:08:32:18.194] samplefrontend backendnodes/node1 0/0/0/0/0 405 501 - - ---- 1/1/0/1/0 0/0 "CONNECT localhost:8080 HTTP/1.1"'
|
226
|
+
end
|
227
|
+
|
228
|
+
it 'matches' do
|
229
|
+
if ecs_compatibility?
|
230
|
+
expect(subject).to include("http"=>hash_including("request"=>{"method"=>"CONNECT"}))
|
231
|
+
expect(subject).to include("url"=>{"port"=>8080, "original"=>"localhost:8080", "domain"=>"localhost"})
|
232
|
+
else
|
233
|
+
expect(subject).to include("http_verb" => "CONNECT")
|
234
|
+
expect(subject).to include("http_host" => "localhost:8080")
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
end
|
239
|
+
|
240
|
+
end
|
241
|
+
|
242
|
+
describe_pattern "HAPROXYTCP", ['legacy', 'ecs-v1'] do
|
243
|
+
|
244
|
+
let(:message) do
|
245
|
+
'Sep 20 15:44:23 127.0.0.1 haproxy[25457]: 127.0.0.1:40962 [20/Sep/2018:15:44:23.285] main app/<NOSRV> -1/-1/1 212 SC 1/1/0/0/0 0/0'
|
246
|
+
end
|
247
|
+
|
248
|
+
it 'matches' do
|
249
|
+
if ecs_compatibility?
|
250
|
+
expect(subject).to include(
|
251
|
+
"timestamp"=>"Sep 20 15:44:23",
|
252
|
+
"host"=>{"hostname"=>"127.0.0.1"},
|
253
|
+
"process"=>{"pid"=>25457, "name"=>"haproxy"},
|
254
|
+
"source"=>{"port"=>40962, "address"=>"127.0.0.1", "bytes"=>212},
|
255
|
+
"haproxy"=>{
|
256
|
+
"request_date"=>"20/Sep/2018:15:44:23.285",
|
257
|
+
"frontend_name"=>"main", "backend_name"=>"app",
|
258
|
+
"total_time_ms"=>"1",
|
259
|
+
"termination_state"=>"SC",
|
260
|
+
"connections"=>{"active"=>1, "backend"=>0, "retries"=>0, "server"=>0, "frontend"=>1},
|
261
|
+
"server_queue"=>0, "backend_queue"=>0
|
262
|
+
})
|
263
|
+
else
|
264
|
+
expect(subject).to include(
|
265
|
+
"syslog_timestamp"=>"Sep 20 15:44:23",
|
266
|
+
"syslog_server"=>"127.0.0.1",
|
267
|
+
"program"=>"haproxy", "pid"=>"25457",
|
268
|
+
"client_ip"=>"127.0.0.1", "client_port"=>"40962",
|
269
|
+
"accept_date"=>"20/Sep/2018:15:44:23.285",
|
270
|
+
"frontend_name"=>"main",
|
271
|
+
"backend_name"=>"app",
|
272
|
+
"server_name"=>"<NOSRV>",
|
273
|
+
"time_backend_connect"=>"-1",
|
274
|
+
"time_queue"=>"-1",
|
275
|
+
"time_duration"=>"1",
|
276
|
+
"bytes_read"=>"212",
|
277
|
+
"termination_state"=>"SC",
|
278
|
+
"actconn"=>"1", "feconn"=>"1", "beconn"=>"0", "backend_queue"=>"0", "retries"=>"0",
|
279
|
+
"srv_queue"=>"0", "srvconn"=>"0",
|
280
|
+
)
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
end
|
data/spec/patterns/httpd_spec.rb
CHANGED
@@ -2,30 +2,51 @@
|
|
2
2
|
require "spec_helper"
|
3
3
|
require "logstash/patterns/core"
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
let(:pattern) { 'HTTPD_COMBINEDLOG' }
|
8
|
-
let(:grok) { grok_match(pattern, message) }
|
5
|
+
describe_pattern "HTTPD_COMBINEDLOG", ['legacy', 'ecs-v1'] do
|
9
6
|
|
10
7
|
context "typical test case" do
|
11
8
|
|
12
9
|
let(:message) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'}
|
13
10
|
|
14
11
|
it "matches" do
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
12
|
+
if ecs_compatibility?
|
13
|
+
expect(grok).to include(
|
14
|
+
"http" => {
|
15
|
+
"request" => {
|
16
|
+
"method" => "GET",
|
17
|
+
"referrer" => "http://semicomplete.com/presentations/logstash-monitorama-2013/"
|
18
|
+
},
|
19
|
+
"response" => {
|
20
|
+
"body" => { "bytes" => 203023 },
|
21
|
+
"status_code" => 200
|
22
|
+
},
|
23
|
+
"version"=>"1.1"
|
24
|
+
},
|
25
|
+
"source" => { "address" => "83.149.9.216" },
|
26
|
+
"url" => { "original" => "/presentations/logstash-monitorama-2013/images/kibana-search.png" },
|
27
|
+
"user_agent" => { "original" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" }
|
28
|
+
)
|
29
|
+
else
|
30
|
+
expect(grok).to include(
|
31
|
+
'clientip' => '83.149.9.216',
|
32
|
+
'verb' => 'GET',
|
33
|
+
'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
|
34
|
+
'httpversion' => '1.1',
|
35
|
+
'response' => '200',
|
36
|
+
'bytes' => '203023',
|
37
|
+
'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
|
38
|
+
'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
|
39
|
+
)
|
40
|
+
end
|
25
41
|
end
|
26
42
|
|
27
43
|
it "does not capture 'null' fields" do
|
28
|
-
|
44
|
+
if ecs_compatibility?
|
45
|
+
expect(grok.keys).to_not include('user') # 'user' => 'name'
|
46
|
+
expect(grok.keys).to_not include('apache') # apache.access.user.identity
|
47
|
+
else
|
48
|
+
expect(grok).to include('auth' => '-', 'ident' => '-')
|
49
|
+
end
|
29
50
|
end
|
30
51
|
|
31
52
|
end
|
@@ -35,7 +56,11 @@ describe "HTTPD_COMBINEDLOG" do
|
|
35
56
|
let(:message) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'}
|
36
57
|
|
37
58
|
it "gets captured" do
|
38
|
-
|
59
|
+
if ecs_compatibility?
|
60
|
+
expect(grok).to include("user" => { 'name' => "username@example.com" })
|
61
|
+
else
|
62
|
+
expect(grok).to include("auth" => "username@example.com")
|
63
|
+
end
|
39
64
|
end
|
40
65
|
|
41
66
|
end
|
@@ -45,121 +70,212 @@ describe "HTTPD_COMBINEDLOG" do
|
|
45
70
|
let(:message) { '83.149.9.216 - a.user [11/Jan/2020:23:05:27 +0100] "OPTIONS /remote.php/ HTTP/1.1" - 7908 "-" "monitoring-client (v2.2)"' }
|
46
71
|
|
47
72
|
it 'matches' do
|
48
|
-
|
73
|
+
if ecs_compatibility?
|
74
|
+
expect(grok).to include("http" => hash_including("response" => hash_including("body" => { "bytes" => 7908 })))
|
75
|
+
expect(grok).to include("http" => hash_including("request" => { "method" => "OPTIONS" }, "version" => "1.1"))
|
76
|
+
expect(grok).to include(
|
77
|
+
"url" => { "original" => "/remote.php/" },
|
78
|
+
"user_agent" => { "original" => "monitoring-client (v2.2)" }
|
79
|
+
)
|
80
|
+
else
|
81
|
+
expect(grok).to include("verb" => "OPTIONS", 'request' => '/remote.php/', 'httpversion' => '1.1', "bytes" => '7908')
|
82
|
+
end
|
49
83
|
end
|
50
84
|
|
51
85
|
it 'does not capture optional response code' do
|
52
|
-
|
86
|
+
if ecs_compatibility?
|
87
|
+
expect(grok['http']['response'].keys).to_not include("status_code")
|
88
|
+
else
|
89
|
+
expect(grok.keys).to_not include("response")
|
90
|
+
end
|
53
91
|
end
|
54
92
|
|
55
93
|
end
|
56
94
|
|
57
95
|
end
|
58
96
|
|
59
|
-
|
60
|
-
|
61
|
-
let(:pattern) { 'HTTPD_ERRORLOG' }
|
62
|
-
let(:grok) { grok_match(pattern, message) }
|
97
|
+
describe_pattern "HTTPD_ERRORLOG", ['legacy', 'ecs-v1'] do
|
63
98
|
|
64
|
-
context "
|
99
|
+
context "a full httpd 2.4 message" do
|
65
100
|
let(:message) do
|
66
101
|
"[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product"
|
67
102
|
end
|
68
|
-
it "generates the fields" do
|
69
103
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
104
|
+
it "generates the fields" do
|
105
|
+
expect(grok).to include('timestamp' => 'Mon Aug 31 09:30:48.958285 2015')
|
106
|
+
if ecs_compatibility?
|
107
|
+
expect(grok).to include("log" => { "level" => "error" })
|
108
|
+
expect(grok).to include("process" => { "pid" => 28787, "thread" => { "id" => 140169587934976 } })
|
109
|
+
expect(grok).to include("source" => { "address"=>"58.13.45.166", "port" => 59307 })
|
110
|
+
expect(grok).to include("error" => { "code" => 'AH01075' })
|
111
|
+
expect(grok).to include("apache" => { "error" => {
|
112
|
+
"module" => "proxy_fcgi",
|
113
|
+
"proxy" => { "error" => { "code" => '70008', "message" => "Partial results are valid but processing is incomplete" }}}
|
114
|
+
})
|
115
|
+
else
|
116
|
+
expect(grok).to include(
|
117
|
+
'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
|
118
|
+
'module' => 'proxy_fcgi',
|
119
|
+
'loglevel' => 'error',
|
120
|
+
'pid' => '28787',
|
121
|
+
'tid' => '140169587934976',
|
122
|
+
'proxy_errorcode' => '70008',
|
123
|
+
'proxy_message' => 'Partial results are valid but processing is incomplete',
|
124
|
+
'clientip' => '58.13.45.166',
|
125
|
+
'clientport' => '59307',
|
126
|
+
'errorcode' => 'AH01075'
|
127
|
+
)
|
128
|
+
end
|
129
|
+
expect(grok).to include('message' => [ message, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ])
|
83
130
|
end
|
84
131
|
end
|
85
132
|
|
86
|
-
context "
|
133
|
+
context "a httpd 2.2 log message" do
|
87
134
|
let(:message) do
|
88
135
|
"[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com"
|
89
136
|
end
|
137
|
+
|
90
138
|
it "generates the fields" do
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
139
|
+
if ecs_compatibility?
|
140
|
+
expect(grok).to include(
|
141
|
+
"timestamp"=>"Mon Aug 31 16:27:04 2015",
|
142
|
+
"log"=>{"level"=>"error"},
|
143
|
+
"source"=>{"address"=>"10.17.42.3"})
|
144
|
+
expect(grok.keys).to_not include("error") # error.code
|
145
|
+
else
|
146
|
+
expect(grok).to include(
|
147
|
+
'timestamp' => 'Mon Aug 31 16:27:04 2015',
|
148
|
+
'loglevel' => 'error',
|
149
|
+
'clientip' => '10.17.42.3'
|
150
|
+
)
|
151
|
+
expect(grok.keys).to_not include('errorcode')
|
152
|
+
end
|
153
|
+
expect(grok).to include('message' => [ message, 'Premature end of script headers: example.com' ])
|
97
154
|
end
|
98
155
|
end
|
99
156
|
|
100
|
-
context "
|
157
|
+
context "a short httpd 2.4 message" do
|
101
158
|
let(:value1) {
|
102
159
|
"[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'"
|
103
160
|
}
|
104
161
|
it "generates the fields" do
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
162
|
+
match_result = grok_match(pattern, value1)
|
163
|
+
expect(match_result).to include('timestamp' => 'Mon Aug 31 07:15:38.664897 2015')
|
164
|
+
if ecs_compatibility?
|
165
|
+
expect(match_result).to include(
|
166
|
+
"apache"=>{"error"=>{"module"=>"proxy_fcgi"}},
|
167
|
+
"log"=>{"level"=>"error"},
|
168
|
+
"process"=>{"pid"=>28786, "thread"=>{"id"=>140169629898496}},
|
169
|
+
"source"=>{"address"=>"81.139.1.34", "port"=>52042},
|
170
|
+
"error"=>{"code"=>"AH01071"},
|
171
|
+
)
|
172
|
+
else
|
173
|
+
expect(match_result).to include(
|
174
|
+
'module' => 'proxy_fcgi',
|
175
|
+
'loglevel' => 'error',
|
176
|
+
'pid' => '28786',
|
177
|
+
'tid' => '140169629898496',
|
178
|
+
'clientip' => '81.139.1.34',
|
179
|
+
'clientport' => '52042',
|
180
|
+
'errorcode' => 'AH01071'
|
181
|
+
)
|
182
|
+
end
|
183
|
+
expect(match_result).to include('message' => [ value1, "Got error 'Primary script unknown\n'" ])
|
116
184
|
end
|
117
185
|
|
118
186
|
let(:value2) {
|
119
187
|
"[Thu Apr 27 10:39:46.719636 2017] [php7:notice] [pid 17] [client 10.255.0.3:49580] Test error log record"
|
120
188
|
}
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
189
|
+
it "generates the fields" do
|
190
|
+
match_result = grok_match(pattern, value2)
|
191
|
+
expect(match_result).to include('timestamp' => 'Thu Apr 27 10:39:46.719636 2017')
|
192
|
+
if ecs_compatibility?
|
193
|
+
expect(match_result).to include(
|
194
|
+
"apache"=>{"error"=>{"module"=>"php7"}},
|
195
|
+
"log"=>{"level"=>"notice"},
|
196
|
+
"process"=>{"pid"=>17},
|
197
|
+
"source"=>{"port"=>49580, "address"=>"10.255.0.3"}
|
198
|
+
)
|
199
|
+
else
|
200
|
+
expect(match_result).to include(
|
201
|
+
'module' => 'php7',
|
202
|
+
'loglevel' => 'notice',
|
203
|
+
'pid' => '17',
|
204
|
+
'clientip' => '10.255.0.3',
|
205
|
+
'clientport' => '49580'
|
206
|
+
)
|
207
|
+
end
|
208
|
+
expect(match_result).to include('message' => [ value2, "Test error log record" ])
|
131
209
|
end
|
132
210
|
end
|
133
211
|
|
134
|
-
context "
|
212
|
+
context "a httpd 2.4 restart message" do
|
135
213
|
let(:value1) {
|
136
214
|
"[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations"
|
137
215
|
}
|
138
216
|
it "generates the fields" do
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
217
|
+
match_result = grok_match(pattern, value1)
|
218
|
+
expect(match_result).to include('timestamp' => 'Mon Aug 31 06:29:47.406518 2015')
|
219
|
+
if ecs_compatibility?
|
220
|
+
expect(match_result).to include(
|
221
|
+
"apache"=>{"error"=>{"module"=>"mpm_event"}},
|
222
|
+
"log"=>{"level"=>"notice"},
|
223
|
+
"process"=>{"pid"=>24968, "thread"=>{"id"=>140169861986176}},
|
224
|
+
"error"=>{"code"=>"AH00489"}
|
225
|
+
)
|
226
|
+
|
227
|
+
else
|
228
|
+
expect(match_result).to include(
|
229
|
+
'module' => 'mpm_event',
|
230
|
+
'loglevel' => 'notice',
|
231
|
+
'pid' => '24968',
|
232
|
+
'tid' => '140169861986176',
|
233
|
+
'errorcode' => 'AH00489'
|
234
|
+
)
|
235
|
+
end
|
236
|
+
expect(match_result).to include('message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ])
|
148
237
|
end
|
149
238
|
|
150
239
|
let(:value2) {
|
151
240
|
"[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'"
|
152
241
|
}
|
153
242
|
it "generates the fields" do
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
243
|
+
match_result = grok_match(pattern, value2)
|
244
|
+
expect(match_result).to include('timestamp' => 'Mon Aug 31 06:29:47.406530 2015')
|
245
|
+
if ecs_compatibility?
|
246
|
+
expect(match_result).to include(
|
247
|
+
"apache"=>{"error"=>{"module"=>"core"}},
|
248
|
+
"log"=>{"level"=>"notice"},
|
249
|
+
"process"=>{"pid"=>24968, "thread"=>{"id"=>140169861986176}},
|
250
|
+
"error"=>{"code"=>"AH00094"}
|
251
|
+
)
|
252
|
+
else
|
253
|
+
expect(match_result).to include(
|
254
|
+
'module' => 'core',
|
255
|
+
'loglevel' => 'notice',
|
256
|
+
'pid' => '24968',
|
257
|
+
'tid' => '140169861986176',
|
258
|
+
'errorcode' => 'AH00094'
|
259
|
+
)
|
260
|
+
end
|
261
|
+
expect(match_result).to include('message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ])
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
context "a httpd 2.4 message witout module" do
|
266
|
+
let(:message) do
|
267
|
+
"[Tue Apr 14 14:27:52.605084 2020] [:error] [pid 5688] [client 192.168.10.110:8196] script '/login/wp-login.php' not found or unable to stat"
|
268
|
+
end
|
269
|
+
|
270
|
+
it "matches" do
|
271
|
+
expect(grok).to include('timestamp' => 'Tue Apr 14 14:27:52.605084 2020')
|
272
|
+
if ecs_compatibility?
|
273
|
+
expect(grok).to include("log"=>{"level" => "error"})
|
274
|
+
expect(grok).to include("process"=>{"pid" => 5688})
|
275
|
+
expect( ((grok['apache'] || {})['error'] || {}).keys ).to_not include('module')
|
276
|
+
else
|
277
|
+
expect(grok).to include('loglevel' => 'error', 'pid' => '5688')
|
278
|
+
end
|
163
279
|
end
|
164
280
|
end
|
165
281
|
|
@@ -169,15 +285,20 @@ describe "HTTPD_ERRORLOG" do
|
|
169
285
|
end
|
170
286
|
|
171
287
|
it 'matches imperfectly (legacy)' do
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
288
|
+
if ecs_compatibility?
|
289
|
+
pending
|
290
|
+
raise NotImplementedError, "TODO: would be nice to 'improve' matching on these debug logs as well"
|
291
|
+
else
|
292
|
+
expect(grok).to include({
|
293
|
+
"timestamp"=>"Fri Feb 01 22:03:08.319124 2019",
|
294
|
+
"module"=>"authz_core",
|
295
|
+
"loglevel"=>"debug",
|
296
|
+
"pid"=>"9",
|
297
|
+
"tid"=>"140597881775872",
|
298
|
+
"errorcode"=>"mod_authz_core.c(820)",
|
299
|
+
"message"=>[message, "[client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted"]
|
300
|
+
})
|
301
|
+
end
|
181
302
|
end
|
182
303
|
end
|
183
304
|
|