logstash-patterns-core 4.2.0 → 4.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +98 -0
- data/Gemfile +3 -0
- data/README.md +11 -18
- data/lib/logstash/patterns/core.rb +11 -3
- data/logstash-patterns-core.gemspec +1 -1
- data/patterns/ecs-v1/aws +28 -0
- data/patterns/ecs-v1/bacula +53 -0
- data/patterns/ecs-v1/bind +13 -0
- data/patterns/ecs-v1/bro +30 -0
- data/patterns/ecs-v1/exim +26 -0
- data/patterns/ecs-v1/firewalls +111 -0
- data/patterns/ecs-v1/grok-patterns +95 -0
- data/patterns/ecs-v1/haproxy +40 -0
- data/patterns/ecs-v1/httpd +17 -0
- data/patterns/ecs-v1/java +34 -0
- data/patterns/ecs-v1/junos +13 -0
- data/patterns/ecs-v1/linux-syslog +16 -0
- data/patterns/{maven → ecs-v1/maven} +0 -0
- data/patterns/ecs-v1/mcollective +4 -0
- data/patterns/ecs-v1/mongodb +7 -0
- data/patterns/ecs-v1/nagios +124 -0
- data/patterns/ecs-v1/postgresql +2 -0
- data/patterns/ecs-v1/rails +13 -0
- data/patterns/ecs-v1/redis +3 -0
- data/patterns/ecs-v1/ruby +2 -0
- data/patterns/ecs-v1/squid +6 -0
- data/patterns/ecs-v1/zeek +33 -0
- data/patterns/{aws → legacy/aws} +1 -1
- data/patterns/{bacula → legacy/bacula} +5 -5
- data/patterns/legacy/bind +3 -0
- data/patterns/{bro → legacy/bro} +0 -0
- data/patterns/{exim → legacy/exim} +8 -2
- data/patterns/{firewalls → legacy/firewalls} +2 -2
- data/patterns/{grok-patterns → legacy/grok-patterns} +0 -0
- data/patterns/{haproxy → legacy/haproxy} +0 -0
- data/patterns/{httpd → legacy/httpd} +1 -1
- data/patterns/{java → legacy/java} +0 -0
- data/patterns/{junos → legacy/junos} +0 -0
- data/patterns/{linux-syslog → legacy/linux-syslog} +0 -0
- data/patterns/legacy/maven +1 -0
- data/patterns/{mcollective → legacy/mcollective} +0 -0
- data/patterns/{mcollective-patterns → legacy/mcollective-patterns} +0 -0
- data/patterns/{mongodb → legacy/mongodb} +0 -0
- data/patterns/{nagios → legacy/nagios} +0 -0
- data/patterns/{postgresql → legacy/postgresql} +0 -0
- data/patterns/{rails → legacy/rails} +0 -0
- data/patterns/{redis → legacy/redis} +0 -0
- data/patterns/{ruby → legacy/ruby} +0 -0
- data/patterns/legacy/squid +4 -0
- data/spec/patterns/aws_spec.rb +395 -0
- data/spec/patterns/bacula_spec.rb +367 -0
- data/spec/patterns/bind_spec.rb +78 -0
- data/spec/patterns/bro_spec.rb +613 -0
- data/spec/patterns/core_spec.rb +51 -9
- data/spec/patterns/exim_spec.rb +201 -0
- data/spec/patterns/firewalls_spec.rb +669 -66
- data/spec/patterns/haproxy_spec.rb +246 -38
- data/spec/patterns/httpd_spec.rb +215 -94
- data/spec/patterns/java_spec.rb +357 -27
- data/spec/patterns/junos_spec.rb +101 -0
- data/spec/patterns/mcollective_spec.rb +35 -0
- data/spec/patterns/mongodb_spec.rb +170 -33
- data/spec/patterns/nagios_spec.rb +296 -79
- data/spec/patterns/netscreen_spec.rb +123 -0
- data/spec/patterns/rails3_spec.rb +87 -29
- data/spec/patterns/redis_spec.rb +157 -121
- data/spec/patterns/shorewall_spec.rb +85 -74
- data/spec/patterns/squid_spec.rb +139 -0
- data/spec/patterns/syslog_spec.rb +266 -22
- data/spec/spec_helper.rb +80 -6
- metadata +64 -28
- data/patterns/bind +0 -3
- data/patterns/squid +0 -4
- data/spec/patterns/bro.rb +0 -126
- data/spec/patterns/s3_spec.rb +0 -173
@@ -13,9 +13,9 @@ BACULA_LOG_NEW_VOLUME Created new Volume \"%{BACULA_VOLUME:volume}\" in catalog.
|
|
13
13
|
BACULA_LOG_NEW_LABEL Labeled new Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\).
|
14
14
|
BACULA_LOG_WROTE_LABEL Wrote label to prelabeled Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE}\" \(%{BACULA_DEVICEPATH}\)
|
15
15
|
BACULA_LOG_NEW_MOUNT New volume \"%{BACULA_VOLUME:volume}\" mounted on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\) at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}.
|
16
|
-
BACULA_LOG_NOOPEN \s
|
17
|
-
BACULA_LOG_NOOPENDIR \s
|
18
|
-
BACULA_LOG_NOSTAT \s
|
16
|
+
BACULA_LOG_NOOPEN \s*Cannot open %{DATA}: ERR=%{GREEDYDATA:berror}
|
17
|
+
BACULA_LOG_NOOPENDIR \s*Could not open directory %{DATA}: ERR=%{GREEDYDATA:berror}
|
18
|
+
BACULA_LOG_NOSTAT \s*Could not stat %{DATA}: ERR=%{GREEDYDATA:berror}
|
19
19
|
BACULA_LOG_NOJOBS There are no more Jobs associated with Volume \"%{BACULA_VOLUME:volume}\". Marking it purged.
|
20
20
|
BACULA_LOG_ALL_RECORDS_PRUNED All records pruned from Volume \"%{BACULA_VOLUME:volume}\"; marking it \"Purged\"
|
21
21
|
BACULA_LOG_BEGIN_PRUNE_JOBS Begin pruning Jobs older than %{INT} month %{INT} days .
|
@@ -41,10 +41,10 @@ BACULA_LOG_DUPLICATE Fatal error: JobId %{INT:duplicate} already running. Duplic
|
|
41
41
|
BACULA_LOG_NOJOBSTAT Fatal error: No Job status returned from FD.
|
42
42
|
BACULA_LOG_FATAL_CONN Fatal error: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=(?<berror>%{GREEDYDATA})
|
43
43
|
BACULA_LOG_NO_CONNECT Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=(?<berror>%{GREEDYDATA})
|
44
|
-
BACULA_LOG_NO_AUTH Fatal error: Unable to authenticate with File daemon at %{
|
44
|
+
BACULA_LOG_NO_AUTH Fatal error: Unable to authenticate with File daemon at %{DATA}. Possible causes:
|
45
45
|
BACULA_LOG_NOSUIT No prior or suitable Full backup found in catalog. Doing FULL backup.
|
46
46
|
BACULA_LOG_NOPRIOR No prior Full backup Job record found.
|
47
47
|
|
48
48
|
BACULA_LOG_JOB (Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\):
|
49
49
|
|
50
|
-
BACULA_LOGLINE %{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname} JobId %{INT:jobid}
|
50
|
+
BACULA_LOGLINE %{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname}(?: JobId %{INT:jobid})?:? (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})
|
@@ -0,0 +1,3 @@
|
|
1
|
+
BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
|
2
|
+
|
3
|
+
BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\)
|
data/patterns/{bro → legacy/bro}
RENAMED
File without changes
|
@@ -1,13 +1,19 @@
|
|
1
1
|
EXIM_MSGID [0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2}
|
2
2
|
EXIM_FLAGS (<=|[-=>*]>|[*]{2}|==)
|
3
3
|
EXIM_DATE %{YEAR:exim_year}-%{MONTHNUM:exim_month}-%{MONTHDAY:exim_day} %{TIME:exim_time}
|
4
|
-
EXIM_PID \[%{POSINT}\]
|
4
|
+
EXIM_PID \[%{POSINT:pid}\]
|
5
5
|
EXIM_QT ((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?)
|
6
6
|
EXIM_EXCLUDE_TERMS (Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message)
|
7
|
-
EXIM_REMOTE_HOST (H=(%{NOTSPACE:remote_hostname} )?(\(%{NOTSPACE:remote_heloname}\) )?\[%{IP:remote_host}\])
|
7
|
+
EXIM_REMOTE_HOST (H=(%{NOTSPACE:remote_hostname} )?(\(%{NOTSPACE:remote_heloname}\) )?\[%{IP:remote_host}\])(?::%{POSINT:remote_port})?
|
8
8
|
EXIM_INTERFACE (I=\[%{IP:exim_interface}\](:%{NUMBER:exim_interface_port}))
|
9
9
|
EXIM_PROTOCOL (P=%{NOTSPACE:protocol})
|
10
10
|
EXIM_MSG_SIZE (S=%{NUMBER:exim_msg_size})
|
11
11
|
EXIM_HEADER_ID (id=%{NOTSPACE:exim_header_id})
|
12
12
|
EXIM_SUBJECT (T=%{QS:exim_subject})
|
13
13
|
|
14
|
+
EXIM_UNKNOWN_FIELD (?:[A-Za-z0-9]{1,4}=%{NOTSPACE})
|
15
|
+
EXIM_NAMED_FIELDS (?: (?:%{EXIM_REMOTE_HOST}|%{EXIM_INTERFACE}|%{EXIM_PROTOCOL}|%{EXIM_MSG_SIZE}|%{EXIM_HEADER_ID}|%{EXIM_SUBJECT}|%{EXIM_UNKNOWN_FIELD}))*
|
16
|
+
|
17
|
+
EXIM_MESSAGE_ARRIVAL %{EXIM_DATE:timestamp} (?:%{EXIM_PID} )?%{EXIM_MSGID:exim_msgid} (?<exim_flags><=) (?<exim_status>[a-z:] )?%{EMAILADDRESS:exim_sender_email}%{EXIM_NAMED_FIELDS}(?: for %{EMAILADDRESS:exim_recipient_email})?
|
18
|
+
|
19
|
+
EXIM %{EXIM_MESSAGE_ARRIVAL}
|
@@ -84,8 +84,8 @@ CISCOFW733100 \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Cu
|
|
84
84
|
#== End Cisco ASA ==
|
85
85
|
|
86
86
|
# Shorewall firewall logs
|
87
|
-
SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host})
|
87
|
+
SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) .*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)
|
88
88
|
#== End Shorewall
|
89
89
|
#== SuSE Firewall 2 ==
|
90
|
-
SFW2 ((%{SYSLOGTIMESTAMP})|(%{TIMESTAMP_ISO8601}))\s*%{HOSTNAME}\s*kernel\S+\s
|
90
|
+
SFW2 ((%{SYSLOGTIMESTAMP:timestamp})|(%{TIMESTAMP_ISO8601:timestamp}))\s*%{HOSTNAME}\s*kernel\S+\s*(?:%{NAGIOSTIME}\s*)?SFW2\-INext\-%{NOTSPACE:nf_action}\s*IN=%{USERNAME:nf_in_interface}.*OUT=(\s*%{USERNAME:nf_out_interface})?\s*MAC=((%{COMMONMAC:nf_dst_mac}:%{COMMONMAC:nf_src_mac})|(\s*)).*SRC=%{IP:nf_src_ip}\s*DST=%{IP:nf_dst_ip}.*PROTO=%{WORD:nf_protocol}((.*SPT=%{INT:nf_src_port}.*DPT=%{INT:nf_dst_port}.*)|())
|
91
91
|
#== End SuSE ==
|
File without changes
|
File without changes
|
@@ -7,7 +7,7 @@ HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}
|
|
7
7
|
|
8
8
|
# Error logs
|
9
9
|
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}
|
10
|
-
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[
|
10
|
+
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[(?:%{WORD:module})?:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}
|
11
11
|
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
|
12
12
|
|
13
13
|
# Deprecated
|
File without changes
|
File without changes
|
File without changes
|
@@ -0,0 +1 @@
|
|
1
|
+
MAVEN_VERSION (?:(\d+)\.)?(?:(\d+)\.)?(\*|\d+)(?:[.-](RELEASE|SNAPSHOT))?
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
@@ -0,0 +1,4 @@
|
|
1
|
+
# Pattern squid3
|
2
|
+
# Documentation of squid3 logs formats can be found at the following link:
|
3
|
+
# http://wiki.squid-cache.org/Features/LogFormat
|
4
|
+
SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:duration}\s%{IP:client_address}\s%{WORD:cache_result}/%{NONNEGINT:status_code}\s%{NUMBER:bytes}\s%{WORD:request_method}\s%{NOTSPACE:url}\s(%{NOTSPACE:user}|-)\s%{WORD:hierarchy_code}/(%{IPORHOST:server}|-)\s%{NOTSPACE:content_type}
|
@@ -0,0 +1,395 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "spec_helper"
|
3
|
+
require "logstash/patterns/core"
|
4
|
+
|
5
|
+
describe_pattern "ELB_ACCESS_LOG", ['legacy', 'ecs-v1'] do
|
6
|
+
|
7
|
+
context "parsing an access log" do
|
8
|
+
|
9
|
+
let(:message) do
|
10
|
+
"2014-02-15T23:39:43.945958Z my-test-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\""
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'matches' do
|
14
|
+
should include("timestamp" => "2014-02-15T23:39:43.945958Z" )
|
15
|
+
if ecs_compatibility?
|
16
|
+
expect(grok).to include("aws" => { "elb" => {
|
17
|
+
"name"=>"my-test-loadbalancer",
|
18
|
+
"request_processing_time"=>{"sec"=>0.000073},
|
19
|
+
"response_processing_time"=>{"sec"=>0.000057},
|
20
|
+
"backend_processing_time"=>{"sec"=>0.001048},
|
21
|
+
"backend"=>{
|
22
|
+
"ip"=>"10.0.0.1", "port"=>80,
|
23
|
+
"http"=>{"response"=>{"status_code"=>200}}
|
24
|
+
}
|
25
|
+
}})
|
26
|
+
expect(grok).to include("http"=>{
|
27
|
+
"request"=>{"body"=>{"bytes"=>0}, "method"=>"GET"},
|
28
|
+
"response"=>{"body"=>{"bytes"=>29}, "status_code"=>200},
|
29
|
+
"version"=>"1.1"
|
30
|
+
})
|
31
|
+
expect(grok).to include("source"=>{"ip"=>"192.168.131.39", "port"=>2817})
|
32
|
+
expect(grok).to include("url"=>{
|
33
|
+
"original"=>"http://www.example.com:80/",
|
34
|
+
"port"=>80, "path"=>"/", "domain"=>"www.example.com", "scheme"=>"http"
|
35
|
+
})
|
36
|
+
else
|
37
|
+
should include("elb" => "my-test-loadbalancer" )
|
38
|
+
should include("clientip" => "192.168.131.39" )
|
39
|
+
should include("clientport" => 2817 )
|
40
|
+
should include("backendip" => "10.0.0.1" )
|
41
|
+
should include("backendport" => 80 )
|
42
|
+
should include("request_processing_time" => 0.000073 )
|
43
|
+
should include("backend_processing_time" => 0.001048 )
|
44
|
+
should include("response_processing_time" => 0.000057 )
|
45
|
+
should include("response" => 200 )
|
46
|
+
should include("backend_response" => 200 )
|
47
|
+
should include("received_bytes" => 0 )
|
48
|
+
should include("bytes" => 29 )
|
49
|
+
should include("verb" => "GET" )
|
50
|
+
should include("request" => "http://www.example.com:80/" )
|
51
|
+
should include("proto" => "http" )
|
52
|
+
should include("httpversion" => "1.1" )
|
53
|
+
should include("urihost" => "www.example.com:80" )
|
54
|
+
should include("path" => "/" )
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
["tags", "params"].each do |attribute|
|
59
|
+
it "have #{attribute} as nil" do
|
60
|
+
expect(subject[attribute]).to be_nil
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
context "parsing a PUT request access log with missing backend info" do
|
66
|
+
|
67
|
+
let(:message) do
|
68
|
+
'2015-04-10T08:11:09.865823Z us-west-1-production-media 49.150.87.133:55128 - -1 -1 -1 408 - 1294336 0 "PUT https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw HTTP/1.1"'
|
69
|
+
end
|
70
|
+
|
71
|
+
it "matches" do
|
72
|
+
expect(grok).to include("timestamp"=>"2015-04-10T08:11:09.865823Z")
|
73
|
+
if ecs_compatibility?
|
74
|
+
expect(grok).to include("url"=>{
|
75
|
+
"original"=>"https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw",
|
76
|
+
"scheme"=>"https", "port"=>443, "path"=>"/videos/F4_M-T4X0MM6Hvy1PFHesw", "domain"=>"media.xxxyyyzzz.com"
|
77
|
+
})
|
78
|
+
expect(grok).to include("source"=>{"port"=>55128, "ip"=>"49.150.87.133"})
|
79
|
+
expect(grok).to include("http"=>{
|
80
|
+
"request"=>{"method"=>"PUT", "body"=>{"bytes"=>1294336}}, "version"=>"1.1",
|
81
|
+
"response"=>{"body"=>{"bytes"=>0}, "status_code"=>408}
|
82
|
+
})
|
83
|
+
# no backend.ip and backend.port
|
84
|
+
# no backend.http.status.code
|
85
|
+
# no request_processing_time.sec and friends
|
86
|
+
expect(grok).to include("aws"=>{"elb"=>{"name"=>"us-west-1-production-media"}})
|
87
|
+
else
|
88
|
+
expect(grok).to include(
|
89
|
+
"elb"=>"us-west-1-production-media",
|
90
|
+
"clientip"=>"49.150.87.133", "clientport"=>55128,
|
91
|
+
"response_processing_time"=>-1.0,
|
92
|
+
"request_processing_time"=>-1.0,
|
93
|
+
"backend_processing_time"=>-1.0,
|
94
|
+
"response"=>408,
|
95
|
+
"received_bytes"=>1294336,
|
96
|
+
"bytes"=>0,
|
97
|
+
"verb"=>"PUT",
|
98
|
+
"request"=>"https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw",
|
99
|
+
"port"=>"443", "proto"=>"https", "path"=>"/videos/F4_M-T4X0MM6Hvy1PFHesw", "urihost"=>"media.xxxyyyzzz.com:443",
|
100
|
+
"httpversion"=>"1.1")
|
101
|
+
|
102
|
+
expect(grok.keys).to_not include("backendip", "backendport", "backendresponse")
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
|
108
|
+
context '(new) https format' do # slightly longer - 3 fields added at the end
|
109
|
+
|
110
|
+
let(:message) do
|
111
|
+
'2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.38.0 (#56-0ef1d4a5)" DHE-RSA-AES128-SHA TLSv1.2'
|
112
|
+
end
|
113
|
+
|
114
|
+
it 'matches (new) suffix fields' do
|
115
|
+
if ecs_compatibility?
|
116
|
+
expect(grok).to include "tls" => { "cipher" => "DHE-RSA-AES128-SHA" }
|
117
|
+
expect(grok).to include "aws" => { "elb" => hash_including("ssl_protocol" => 'TLSv1.2')}
|
118
|
+
expect(grok).to include "user_agent"=>{"original"=>"curl/7.38.0 (#56-0ef1d4a5)"}
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
context 'with optional fields' do
|
123
|
+
|
124
|
+
let(:message) do
|
125
|
+
'2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "-" - -'
|
126
|
+
end
|
127
|
+
|
128
|
+
it 'matches (new) suffix fields' do
|
129
|
+
if ecs_compatibility?
|
130
|
+
expect(grok.keys).to_not include "tls"
|
131
|
+
expect(grok['aws']['elb'].keys).to_not include "ssl_protocol"
|
132
|
+
expect(grok.keys).to_not include "user_agent"
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
describe_pattern "S3_ACCESS_LOG", ['legacy', 'ecs-v1'] do
|
142
|
+
|
143
|
+
context "parsing GET.VERSIONING message" do
|
144
|
+
|
145
|
+
let(:message) do
|
146
|
+
"79a5 mybucket [06/Feb/2014:00:00:38 +0000] 192.0.2.3 79a5 3E57427F3EXAMPLE REST.GET.VERSIONING - \"GET /mybucket?versioning HTTP/1.1\" 200 - 113 - 7 - \"-\" \"S3Console/0.4\" -"
|
147
|
+
end
|
148
|
+
|
149
|
+
it { should include("owner" => "79a5" ) unless ecs_compatibility? }
|
150
|
+
it { should include("bucket" => "mybucket" ) unless ecs_compatibility? }
|
151
|
+
it { should include("timestamp" => "06/Feb/2014:00:00:38 +0000" ) unless ecs_compatibility? }
|
152
|
+
it { should include("clientip" => "192.0.2.3" ) unless ecs_compatibility? }
|
153
|
+
it { should include("requester" => "79a5" ) unless ecs_compatibility? }
|
154
|
+
it { should include("request_id" => "3E57427F3EXAMPLE" ) unless ecs_compatibility? }
|
155
|
+
it { should include("operation" => "REST.GET.VERSIONING" ) unless ecs_compatibility? }
|
156
|
+
it { should include("key" => "-" ) unless ecs_compatibility? }
|
157
|
+
|
158
|
+
it { should include("verb" => "GET" ) unless ecs_compatibility? }
|
159
|
+
it { should include("request" => "/mybucket?versioning" ) unless ecs_compatibility? }
|
160
|
+
it { should include("httpversion" => "1.1" ) unless ecs_compatibility? }
|
161
|
+
it { should include("response" => 200 ) unless ecs_compatibility? }
|
162
|
+
it { should include("bytes" => 113 ) unless ecs_compatibility? }
|
163
|
+
|
164
|
+
it { should include("request_time_ms" => 7 ) unless ecs_compatibility? }
|
165
|
+
it { should include("referrer" => "\"-\"" ) unless ecs_compatibility? }
|
166
|
+
it { should include("agent" => "\"S3Console/0.4\"" ) unless ecs_compatibility? }
|
167
|
+
|
168
|
+
["tags", "error_code", "object_size", "turnaround_time_ms", "version_id"].each do |attribute|
|
169
|
+
it "have #{attribute} as nil" do
|
170
|
+
expect(subject[attribute]).to be_nil unless ecs_compatibility?
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
end
|
175
|
+
|
176
|
+
context "parsing a GET.OBJECT message" do
|
177
|
+
|
178
|
+
let(:message) do
|
179
|
+
"79a5 mybucket [12/May/2014:07:54:01 +0000] 10.0.1.2 - 7ACC4BE89EXAMPLE REST.GET.OBJECT foo/bar.html \"GET /foo/bar.html HTTP/1.1\" 304 - - 1718 10 - \"-\" \"Mozilla/5.0\" -"
|
180
|
+
end
|
181
|
+
|
182
|
+
it do
|
183
|
+
if ecs_compatibility?
|
184
|
+
should include("aws"=>{"s3access"=>hash_including("bucket_owner" => "79a5")})
|
185
|
+
else
|
186
|
+
should include("owner" => "79a5")
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
it { should include("bucket" => "mybucket" ) unless ecs_compatibility? }
|
191
|
+
it { should include("timestamp" => "12/May/2014:07:54:01 +0000" ) }
|
192
|
+
|
193
|
+
it { should include("clientip" => "10.0.1.2" ) unless ecs_compatibility? }
|
194
|
+
it { should include("requester" => "-" ) unless ecs_compatibility? }
|
195
|
+
it { should include("client" => { 'ip' => '10.0.1.2' } ) if ecs_compatibility? }
|
196
|
+
|
197
|
+
it { should include("request_id" => "7ACC4BE89EXAMPLE" ) unless ecs_compatibility? }
|
198
|
+
it { should include("operation" => "REST.GET.OBJECT" ) unless ecs_compatibility? }
|
199
|
+
|
200
|
+
it do
|
201
|
+
if ecs_compatibility?
|
202
|
+
should include("aws"=>{"s3access"=>hash_including("key" => "foo/bar.html")})
|
203
|
+
else
|
204
|
+
should include("key" => "foo/bar.html")
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
it { should include("verb" => "GET" ) unless ecs_compatibility? }
|
209
|
+
it { should include("request" => "/foo/bar.html" ) unless ecs_compatibility? }
|
210
|
+
it { should include("httpversion" => "1.1" ) unless ecs_compatibility? }
|
211
|
+
it { should include("response" => 304 ) unless ecs_compatibility? }
|
212
|
+
it { should include("object_size" => 1718 ) unless ecs_compatibility? }
|
213
|
+
|
214
|
+
it { should include("request_time_ms" => 10 ) unless ecs_compatibility? }
|
215
|
+
it { should include("referrer" => "\"-\"" ) unless ecs_compatibility? }
|
216
|
+
|
217
|
+
it { should include("agent" => "\"Mozilla/5.0\"" ) unless ecs_compatibility? }
|
218
|
+
it { should include("user_agent"=>{"original"=>"Mozilla/5.0"}) if ecs_compatibility? }
|
219
|
+
|
220
|
+
["tags", "error_code", "turnaround_time_ms", "version_id", "bytes"].each do |attribute|
|
221
|
+
it "have #{attribute} as nil" do
|
222
|
+
expect(subject[attribute]).to be_nil unless ecs_compatibility?
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
end
|
227
|
+
|
228
|
+
context 'a long line' do
|
229
|
+
|
230
|
+
let(:message) do
|
231
|
+
'79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket1 [06/Feb/2019:00:00:38 +0000] ' +
|
232
|
+
'192.0.2.3 arn:aws:iam::123456:user/test@elastic.co A1206F460EXAMPLE REST.GET.BUCKETPOLICY - ' +
|
233
|
+
'"GET /awsexamplebucket1?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 12 "-" ' +
|
234
|
+
'"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64" - ' +
|
235
|
+
'BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 ' +
|
236
|
+
'AuthHeader awsexamplebucket1.s3.us-west-1.amazonaws.com TLSV1.1'
|
237
|
+
end
|
238
|
+
|
239
|
+
it 'matches' do
|
240
|
+
if ecs_compatibility?
|
241
|
+
expect(grok).to include("client"=>{"ip"=>"192.0.2.3", "user"=>{"id"=>"arn:aws:iam::123456:user/test@elastic.co"}})
|
242
|
+
expect(grok).to include("http"=>{"request"=>{"method"=>"GET"}, "version"=>"1.1", "response"=>{"status_code"=>404}})
|
243
|
+
expect(grok).to include("url"=>{"original"=>"/awsexamplebucket1?policy"})
|
244
|
+
expect(grok).to include("aws"=>{"s3access"=>{
|
245
|
+
"bucket_owner"=>"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
|
246
|
+
"bucket"=>"awsexamplebucket1",
|
247
|
+
"request_id"=>"A1206F460EXAMPLE",
|
248
|
+
"operation"=>"REST.GET.BUCKETPOLICY",
|
249
|
+
"turn_around_time"=>12,
|
250
|
+
"bytes_sent"=>297,
|
251
|
+
"request_uri"=>"GET /awsexamplebucket1?policy HTTP/1.1", # NOTE: redundant (beats compatibility)
|
252
|
+
"error_code"=>"NoSuchBucketPolicy",
|
253
|
+
"total_time" => 38,
|
254
|
+
# these fields weren't matched in legacy mode:
|
255
|
+
# Host Id -> Signature Version -> Cipher Suite -> Authentication Type -> Host Header -> TLS version
|
256
|
+
"host_id" => "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=",
|
257
|
+
"signature_version" => "SigV2",
|
258
|
+
#"cipher_suite" => "ECDHE-RSA-AES128-GCM-SHA256", # tls.cipher
|
259
|
+
"authentication_type" => "AuthHeader",
|
260
|
+
"host_header" => "awsexamplebucket1.s3.us-west-1.amazonaws.com",
|
261
|
+
"tls_version" => "TLSV1.1"
|
262
|
+
}})
|
263
|
+
expect(grok).to include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
|
264
|
+
expect(grok).to include("user_agent"=>{
|
265
|
+
"original"=>"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64"
|
266
|
+
})
|
267
|
+
else
|
268
|
+
expect(grok).to include("owner"=>"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
|
269
|
+
"bucket"=>"awsexamplebucket1",
|
270
|
+
"timestamp"=>"06/Feb/2019:00:00:38 +0000",
|
271
|
+
"clientip"=>"192.0.2.3",
|
272
|
+
"requester"=>"arn:aws:iam::123456:user/test@elastic.co",
|
273
|
+
"request_id"=>"A1206F460EXAMPLE",
|
274
|
+
"operation"=>"REST.GET.BUCKETPOLICY",
|
275
|
+
"key"=>"-",
|
276
|
+
"verb"=>"GET",
|
277
|
+
"request"=>"/awsexamplebucket1?policy",
|
278
|
+
"httpversion"=>"1.1",
|
279
|
+
"response"=>404,
|
280
|
+
"error_code"=>"NoSuchBucketPolicy",
|
281
|
+
"bytes"=>297,
|
282
|
+
# object_size nil
|
283
|
+
"request_time_ms"=>38,
|
284
|
+
"turnaround_time_ms"=>12,
|
285
|
+
"referrer"=>"\"-\"",
|
286
|
+
"agent"=>"\"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64\"")
|
287
|
+
end
|
288
|
+
end
|
289
|
+
|
290
|
+
end
|
291
|
+
end
|
292
|
+
|
293
|
+
describe_pattern "CLOUDFRONT_ACCESS_LOG", ['legacy', 'ecs-v1'] do
|
294
|
+
|
295
|
+
let(:message) do
|
296
|
+
"2016-06-10 18:41:39 IAD53 224281 192.168.1.1 GET d27enomp470abc.cloudfront.net /content/sample/thing.pdf 200 https://example.com/ Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36 - - Miss UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg== host.example.com https 883 0.036 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 RefreshHit"
|
297
|
+
end
|
298
|
+
|
299
|
+
it 'matches' do
|
300
|
+
should include("timestamp" => "2016-06-10 18:41:39" )
|
301
|
+
|
302
|
+
if ecs_compatibility?
|
303
|
+
should include("aws"=>{"cloudfront"=>hash_including("x_edge_location"=>"IAD53")})
|
304
|
+
should include("destination"=>{"bytes"=>224281})
|
305
|
+
should include("source"=>{"ip"=>"192.168.1.1", "bytes"=>883})
|
306
|
+
should include("http"=>{
|
307
|
+
"request"=>{"method"=>"GET", "referrer"=>"https://example.com/"},
|
308
|
+
"response"=>{"status_code"=>200}
|
309
|
+
})
|
310
|
+
should include("user_agent"=>{"original"=>"Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36"})
|
311
|
+
should include("url"=>{"domain"=>"d27enomp470abc.cloudfront.net", "path"=>"/content/sample/thing.pdf"})
|
312
|
+
should include("aws"=>{"cloudfront"=>hash_including("x_edge_result_type"=>"Miss")})
|
313
|
+
should include("aws"=>{"cloudfront"=>hash_including("x_edge_request_id"=>'UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==')})
|
314
|
+
should include("network"=>{"protocol"=>"https"})
|
315
|
+
should include("aws"=>{"cloudfront"=>hash_including("http"=>{"request"=>{"host"=>"host.example.com"}})})
|
316
|
+
should include("aws"=>{"cloudfront"=>hash_including("time_taken"=>0.036)})
|
317
|
+
should include("aws"=>{"cloudfront"=>hash_including("ssl_protocol"=>"TLSv1.2")})
|
318
|
+
should include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
|
319
|
+
should include("aws"=>{"cloudfront"=>hash_including("x_edge_response_result_type"=>"RefreshHit")})
|
320
|
+
else
|
321
|
+
should include("x_edge_location" => "IAD53" )
|
322
|
+
should include("sc_bytes" => 224281 )
|
323
|
+
should include("clientip" => "192.168.1.1" )
|
324
|
+
should include("cs_method" => "GET" )
|
325
|
+
should include("cs_host" => "d27enomp470abc.cloudfront.net" )
|
326
|
+
should include("cs_uri_stem" => "/content/sample/thing.pdf" )
|
327
|
+
should include("sc_status" => 200 )
|
328
|
+
should include("referrer" => "https://example.com/" )
|
329
|
+
should include("agent" => "Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36" )
|
330
|
+
should include("cs_uri_query" => "-" )
|
331
|
+
should include("cookies" => "-" )
|
332
|
+
should include("x_edge_result_type" => "Miss" )
|
333
|
+
should include("x_edge_request_id" => "UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==" )
|
334
|
+
should include("x_host_header" => "host.example.com" )
|
335
|
+
should include("cs_protocol" => "https" )
|
336
|
+
should include("cs_bytes" => 883 )
|
337
|
+
should include("time_taken" => 0.036 )
|
338
|
+
should include("x_forwarded_for" => "-" )
|
339
|
+
should include("ssl_protocol" => "TLSv1.2" )
|
340
|
+
should include("ssl_cipher" => "ECDHE-RSA-AES128-GCM-SHA256" )
|
341
|
+
should include("x_edge_response_result_type" => "RefreshHit" )
|
342
|
+
end
|
343
|
+
end
|
344
|
+
|
345
|
+
["tags", "params"].each do |attribute|
|
346
|
+
it "have #{attribute} as nil" do
|
347
|
+
expect(subject[attribute]).to be_nil
|
348
|
+
end
|
349
|
+
end
|
350
|
+
|
351
|
+
context 'version 1.0' do # more fields at the end
|
352
|
+
|
353
|
+
let(:message) do
|
354
|
+
# Version: 1.0 - https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
|
355
|
+
# Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end
|
356
|
+
"2019-12-04 21:02:31 LAX1 392 192.0.2.100 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ== d111111abcdef8.cloudfront.net https 23 0.001 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.001 Hit text/html 78 - -"
|
357
|
+
end
|
358
|
+
|
359
|
+
it 'matches' do
|
360
|
+
should include("timestamp" => "2019-12-04\t21:02:31")
|
361
|
+
|
362
|
+
if ecs_compatibility?
|
363
|
+
should include("destination"=>{"bytes"=>392}, "source"=>{"ip"=>"192.0.2.100", "bytes"=>23, "port"=>11040}) # source.port not matched in legacy mode
|
364
|
+
should include("url"=>{"domain"=>"d111111abcdef8.cloudfront.net", "path"=>"/index.html"})
|
365
|
+
should include("http"=>hash_including("request"=>{"mime_type"=>"text/html", "method"=>"GET"}, "response"=>{"status_code"=>200})) # mime_type not matched in legacy mode
|
366
|
+
should include("user_agent"=>{"original"=>"Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36"})
|
367
|
+
should include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
|
368
|
+
should include("network"=>{"protocol"=>"https"})
|
369
|
+
|
370
|
+
should include("http"=>hash_including("version"=>"2.0"))
|
371
|
+
should include("aws"=>{"cloudfront"=>{
|
372
|
+
"x_edge_location"=>"LAX1",
|
373
|
+
"x_edge_request_id"=>"SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==", # event.id
|
374
|
+
"x_edge_result_type"=>"Hit",
|
375
|
+
"x_edge_response_result_type"=>"Hit",
|
376
|
+
"x_edge_detailed_result_type"=>"Hit", # not captured in legacy mode
|
377
|
+
"time_taken"=>0.001,
|
378
|
+
"time_to_first_byte"=>0.001, # not captured in legacy mode
|
379
|
+
"http"=>{"request"=>{"host"=>"d111111abcdef8.cloudfront.net", "size"=>78}}, # http.request.size not captured in legacy mode
|
380
|
+
"ssl_protocol"=>"TLSv1.2",
|
381
|
+
}})
|
382
|
+
else
|
383
|
+
should include("cs_method"=>"GET", "cs_host"=>"d111111abcdef8.cloudfront.net", "cs_uri_stem"=>"/index.html", "cs_protocol"=>"https", "cs_bytes"=>23)
|
384
|
+
should include("x_host_header"=>"d111111abcdef8.cloudfront.net")
|
385
|
+
should include("time_taken"=>0.001)
|
386
|
+
should include("x_edge_request_id"=>"SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==")
|
387
|
+
should include("agent"=>"Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36")
|
388
|
+
|
389
|
+
should include("x_forwarded_for"=>"text/html") # TODO the legacy pattern does not handle the long(er) format correctly
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
end
|
394
|
+
|
395
|
+
end
|