logstash-patterns-core 4.2.0 → 4.3.0

data/patterns/{bacula → legacy/bacula}

@@ -13,9 +13,9 @@ BACULA_LOG_NEW_VOLUME Created new Volume \"%{BACULA_VOLUME:volume}\" in catalog.
 BACULA_LOG_NEW_LABEL Labeled new Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\).
 BACULA_LOG_WROTE_LABEL Wrote label to prelabeled Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE}\" \(%{BACULA_DEVICEPATH}\)
 BACULA_LOG_NEW_MOUNT New volume \"%{BACULA_VOLUME:volume}\" mounted on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\) at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}.
-BACULA_LOG_NOOPEN \s+Cannot open %{DATA}: ERR=%{GREEDYDATA:berror}
-BACULA_LOG_NOOPENDIR \s+Could not open directory %{DATA}: ERR=%{GREEDYDATA:berror}
-BACULA_LOG_NOSTAT \s+Could not stat %{DATA}: ERR=%{GREEDYDATA:berror}
+BACULA_LOG_NOOPEN \s*Cannot open %{DATA}: ERR=%{GREEDYDATA:berror}
+BACULA_LOG_NOOPENDIR \s*Could not open directory %{DATA}: ERR=%{GREEDYDATA:berror}
+BACULA_LOG_NOSTAT \s*Could not stat %{DATA}: ERR=%{GREEDYDATA:berror}
 BACULA_LOG_NOJOBS There are no more Jobs associated with Volume \"%{BACULA_VOLUME:volume}\". Marking it purged.
 BACULA_LOG_ALL_RECORDS_PRUNED All records pruned from Volume \"%{BACULA_VOLUME:volume}\"; marking it \"Purged\"
 BACULA_LOG_BEGIN_PRUNE_JOBS Begin pruning Jobs older than %{INT} month %{INT} days .
@@ -41,10 +41,10 @@ BACULA_LOG_DUPLICATE Fatal error: JobId %{INT:duplicate} already running. Duplic
 BACULA_LOG_NOJOBSTAT Fatal error: No Job status returned from FD.
 BACULA_LOG_FATAL_CONN Fatal error: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=(?<berror>%{GREEDYDATA})
 BACULA_LOG_NO_CONNECT Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=(?<berror>%{GREEDYDATA})
-BACULA_LOG_NO_AUTH Fatal error: Unable to authenticate with File daemon at %{HOSTNAME}. Possible causes:
+BACULA_LOG_NO_AUTH Fatal error: Unable to authenticate with File daemon at %{DATA}. Possible causes:
 BACULA_LOG_NOSUIT No prior or suitable Full backup found in catalog. Doing FULL backup.
 BACULA_LOG_NOPRIOR No prior Full backup Job record found.
 
 BACULA_LOG_JOB (Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\):
 
-BACULA_LOGLINE %{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname} JobId %{INT:jobid}: (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})
+BACULA_LOGLINE %{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname}(?: JobId %{INT:jobid})?:? (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})

data/patterns/legacy/bind

@@ -0,0 +1,3 @@
+BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
+
+BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\)

data/patterns/{exim → legacy/exim}

@@ -1,13 +1,19 @@
 EXIM_MSGID [0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2}
 EXIM_FLAGS (<=|[-=>*]>|[*]{2}|==)
 EXIM_DATE %{YEAR:exim_year}-%{MONTHNUM:exim_month}-%{MONTHDAY:exim_day} %{TIME:exim_time}
-EXIM_PID \[%{POSINT}\]
+EXIM_PID \[%{POSINT:pid}\]
 EXIM_QT ((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?)
 EXIM_EXCLUDE_TERMS (Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message)
-EXIM_REMOTE_HOST (H=(%{NOTSPACE:remote_hostname} )?(\(%{NOTSPACE:remote_heloname}\) )?\[%{IP:remote_host}\])
+EXIM_REMOTE_HOST (H=(%{NOTSPACE:remote_hostname} )?(\(%{NOTSPACE:remote_heloname}\) )?\[%{IP:remote_host}\])(?::%{POSINT:remote_port})?
 EXIM_INTERFACE (I=\[%{IP:exim_interface}\](:%{NUMBER:exim_interface_port}))
 EXIM_PROTOCOL (P=%{NOTSPACE:protocol})
 EXIM_MSG_SIZE (S=%{NUMBER:exim_msg_size})
 EXIM_HEADER_ID (id=%{NOTSPACE:exim_header_id})
 EXIM_SUBJECT (T=%{QS:exim_subject})
 
+EXIM_UNKNOWN_FIELD (?:[A-Za-z0-9]{1,4}=%{NOTSPACE})
+EXIM_NAMED_FIELDS (?: (?:%{EXIM_REMOTE_HOST}|%{EXIM_INTERFACE}|%{EXIM_PROTOCOL}|%{EXIM_MSG_SIZE}|%{EXIM_HEADER_ID}|%{EXIM_SUBJECT}|%{EXIM_UNKNOWN_FIELD}))*
+
+EXIM_MESSAGE_ARRIVAL %{EXIM_DATE:timestamp} (?:%{EXIM_PID} )?%{EXIM_MSGID:exim_msgid} (?<exim_flags><=) (?<exim_status>[a-z:] )?%{EMAILADDRESS:exim_sender_email}%{EXIM_NAMED_FIELDS}(?: for %{EMAILADDRESS:exim_recipient_email})?
+
+EXIM %{EXIM_MESSAGE_ARRIVAL}

data/patterns/{firewalls → legacy/firewalls}

@@ -84,8 +84,8 @@ CISCOFW733100 \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Cu
 #== End Cisco ASA ==
 
 # Shorewall firewall logs
-SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)
+SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) .*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)
 #== End Shorewall
 #== SuSE Firewall 2 ==
-SFW2 ((%{SYSLOGTIMESTAMP})|(%{TIMESTAMP_ISO8601}))\s*%{HOSTNAME}\s*kernel\S+\s*%{NAGIOSTIME}\s*SFW2\-INext\-%{NOTSPACE:nf_action}\s*IN=%{USERNAME:nf_in_interface}.*OUT=((\s*%{USERNAME:nf_out_interface})|(\s*))MAC=((%{COMMONMAC:nf_dst_mac}:%{COMMONMAC:nf_src_mac})|(\s*)).*SRC=%{IP:nf_src_ip}\s*DST=%{IP:nf_dst_ip}.*PROTO=%{WORD:nf_protocol}((.*SPT=%{INT:nf_src_port}.*DPT=%{INT:nf_dst_port}.*)|())
+SFW2 ((%{SYSLOGTIMESTAMP:timestamp})|(%{TIMESTAMP_ISO8601:timestamp}))\s*%{HOSTNAME}\s*kernel\S+\s*(?:%{NAGIOSTIME}\s*)?SFW2\-INext\-%{NOTSPACE:nf_action}\s*IN=%{USERNAME:nf_in_interface}.*OUT=(\s*%{USERNAME:nf_out_interface})?\s*MAC=((%{COMMONMAC:nf_dst_mac}:%{COMMONMAC:nf_src_mac})|(\s*)).*SRC=%{IP:nf_src_ip}\s*DST=%{IP:nf_dst_ip}.*PROTO=%{WORD:nf_protocol}((.*SPT=%{INT:nf_src_port}.*DPT=%{INT:nf_dst_port}.*)|())
 #== End SuSE ==

data/patterns/{httpd → legacy/httpd}

@@ -7,7 +7,7 @@ HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}
 
 # Error logs
 HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}
-HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}
+HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[(?:%{WORD:module})?:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}
 HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
 
 # Deprecated

data/patterns/legacy/maven

@@ -0,0 +1 @@
+MAVEN_VERSION (?:(\d+)\.)?(?:(\d+)\.)?(\*|\d+)(?:[.-](RELEASE|SNAPSHOT))?

data/patterns/legacy/squid

@@ -0,0 +1,4 @@
+# Pattern squid3
+# Documentation of squid3 logs formats can be found at the following link:
+# http://wiki.squid-cache.org/Features/LogFormat
+SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:duration}\s%{IP:client_address}\s%{WORD:cache_result}/%{NONNEGINT:status_code}\s%{NUMBER:bytes}\s%{WORD:request_method}\s%{NOTSPACE:url}\s(%{NOTSPACE:user}|-)\s%{WORD:hierarchy_code}/(%{IPORHOST:server}|-)\s%{NOTSPACE:content_type}

data/spec/patterns/aws_spec.rb

@@ -0,0 +1,395 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe_pattern "ELB_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  context "parsing an access log" do
+
+    let(:message) do
+      "2014-02-15T23:39:43.945958Z my-test-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\""
+    end
+
+    it 'matches' do
+      should include("timestamp" => "2014-02-15T23:39:43.945958Z" )
+      if ecs_compatibility?
+        expect(grok).to include("aws" => { "elb" => {
+            "name"=>"my-test-loadbalancer",
+            "request_processing_time"=>{"sec"=>0.000073},
+            "response_processing_time"=>{"sec"=>0.000057},
+            "backend_processing_time"=>{"sec"=>0.001048},
+            "backend"=>{
+                "ip"=>"10.0.0.1", "port"=>80,
+                "http"=>{"response"=>{"status_code"=>200}}
+            }
+        }})
+        expect(grok).to include("http"=>{
+            "request"=>{"body"=>{"bytes"=>0}, "method"=>"GET"},
+            "response"=>{"body"=>{"bytes"=>29}, "status_code"=>200},
+            "version"=>"1.1"
+        })
+        expect(grok).to include("source"=>{"ip"=>"192.168.131.39", "port"=>2817})
+        expect(grok).to include("url"=>{
+            "original"=>"http://www.example.com:80/",
+            "port"=>80, "path"=>"/", "domain"=>"www.example.com", "scheme"=>"http"
+        })
+      else
+        should include("elb" => "my-test-loadbalancer" )
+        should include("clientip" => "192.168.131.39" )
+        should include("clientport" => 2817 )
+        should include("backendip" => "10.0.0.1" )
+        should include("backendport" => 80 )
+        should include("request_processing_time" => 0.000073 )
+        should include("backend_processing_time" => 0.001048 )
+        should include("response_processing_time" => 0.000057 )
+        should include("response" => 200 )
+        should include("backend_response" => 200 )
+        should include("received_bytes" => 0 )
+        should include("bytes" => 29 )
+        should include("verb" => "GET" )
+        should include("request" => "http://www.example.com:80/" )
+        should include("proto" => "http" )
+        should include("httpversion" => "1.1" )
+        should include("urihost" => "www.example.com:80" )
+        should include("path" => "/" )
+      end
+    end
+
+    ["tags", "params"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil
+      end
+    end
+  end
+
+  context "parsing a PUT request access log with missing backend info" do
+
+    let(:message) do
+      '2015-04-10T08:11:09.865823Z us-west-1-production-media 49.150.87.133:55128 - -1 -1 -1 408 - 1294336 0 "PUT https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw HTTP/1.1"'
+    end
+
+    it "matches" do
+      expect(grok).to include("timestamp"=>"2015-04-10T08:11:09.865823Z")
+      if ecs_compatibility?
+        expect(grok).to include("url"=>{
+            "original"=>"https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw",
+            "scheme"=>"https", "port"=>443, "path"=>"/videos/F4_M-T4X0MM6Hvy1PFHesw", "domain"=>"media.xxxyyyzzz.com"
+        })
+        expect(grok).to include("source"=>{"port"=>55128, "ip"=>"49.150.87.133"})
+        expect(grok).to include("http"=>{
+            "request"=>{"method"=>"PUT", "body"=>{"bytes"=>1294336}}, "version"=>"1.1",
+            "response"=>{"body"=>{"bytes"=>0}, "status_code"=>408}
+        })
+        # no backend.ip and backend.port
+        # no backend.http.status.code
+        # no request_processing_time.sec and friends
+        expect(grok).to include("aws"=>{"elb"=>{"name"=>"us-west-1-production-media"}})
+      else
+        expect(grok).to include(
+            "elb"=>"us-west-1-production-media",
+            "clientip"=>"49.150.87.133", "clientport"=>55128,
+            "response_processing_time"=>-1.0,
+            "request_processing_time"=>-1.0,
+            "backend_processing_time"=>-1.0,
+            "response"=>408,
+            "received_bytes"=>1294336,
+            "bytes"=>0,
+            "verb"=>"PUT",
+            "request"=>"https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw",
+            "port"=>"443", "proto"=>"https", "path"=>"/videos/F4_M-T4X0MM6Hvy1PFHesw", "urihost"=>"media.xxxyyyzzz.com:443",
+            "httpversion"=>"1.1")
+
+        expect(grok.keys).to_not include("backendip", "backendport", "backendresponse")
+      end
+    end
+
+  end
+
+  context '(new) https format' do # slightly longer - 3 fields added at the end
+
+    let(:message) do
+      '2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.38.0 (#56-0ef1d4a5)" DHE-RSA-AES128-SHA TLSv1.2'
+    end
+
+    it 'matches (new) suffix fields' do
+      if ecs_compatibility?
+        expect(grok).to include "tls" => { "cipher" => "DHE-RSA-AES128-SHA" }
+        expect(grok).to include "aws" => { "elb" => hash_including("ssl_protocol" => 'TLSv1.2')}
+        expect(grok).to include "user_agent"=>{"original"=>"curl/7.38.0 (#56-0ef1d4a5)"}
+      end
+    end
+
+    context 'with optional fields' do
+
+      let(:message) do
+        '2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "-" - -'
+      end
+
+      it 'matches (new) suffix fields' do
+        if ecs_compatibility?
+          expect(grok.keys).to_not include "tls"
+          expect(grok['aws']['elb'].keys).to_not include "ssl_protocol"
+          expect(grok.keys).to_not include "user_agent"
+        end
+      end
+
+    end
+
+  end
+end
+
+describe_pattern "S3_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  context "parsing GET.VERSIONING message" do
+
+    let(:message) do
+      "79a5 mybucket [06/Feb/2014:00:00:38 +0000] 192.0.2.3 79a5 3E57427F3EXAMPLE REST.GET.VERSIONING - \"GET /mybucket?versioning HTTP/1.1\" 200 - 113 - 7 - \"-\" \"S3Console/0.4\" -"
+    end
+
+    it { should include("owner" => "79a5" ) unless ecs_compatibility? }
+    it { should include("bucket" => "mybucket" ) unless ecs_compatibility? }
+    it { should include("timestamp" => "06/Feb/2014:00:00:38 +0000" ) unless ecs_compatibility? }
+    it { should include("clientip" => "192.0.2.3" ) unless ecs_compatibility? }
+    it { should include("requester" => "79a5" ) unless ecs_compatibility? }
+    it { should include("request_id" => "3E57427F3EXAMPLE" ) unless ecs_compatibility? }
+    it { should include("operation" => "REST.GET.VERSIONING" ) unless ecs_compatibility? }
+    it { should include("key" => "-" ) unless ecs_compatibility? }
+
+    it { should include("verb" => "GET" ) unless ecs_compatibility? }
+    it { should include("request" => "/mybucket?versioning" ) unless ecs_compatibility? }
+    it { should include("httpversion" => "1.1" ) unless ecs_compatibility? }
+    it { should include("response" => 200 ) unless ecs_compatibility? }
+    it { should include("bytes" => 113 ) unless ecs_compatibility? }
+
+    it { should include("request_time_ms" => 7 ) unless ecs_compatibility? }
+    it { should include("referrer" => "\"-\"" ) unless ecs_compatibility? }
+    it { should include("agent" => "\"S3Console/0.4\"" ) unless ecs_compatibility? }
+
+    ["tags", "error_code", "object_size", "turnaround_time_ms", "version_id"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil unless ecs_compatibility?
+      end
+    end
+
+  end
+
+  context "parsing a GET.OBJECT message" do
+
+    let(:message) do
+      "79a5 mybucket [12/May/2014:07:54:01 +0000] 10.0.1.2 - 7ACC4BE89EXAMPLE REST.GET.OBJECT foo/bar.html \"GET /foo/bar.html HTTP/1.1\" 304 - - 1718 10 - \"-\" \"Mozilla/5.0\" -"
+    end
+
+    it do
+      if ecs_compatibility?
+        should include("aws"=>{"s3access"=>hash_including("bucket_owner" => "79a5")})
+      else
+        should include("owner" => "79a5")
+      end
+    end
+
+    it { should include("bucket" => "mybucket" ) unless ecs_compatibility? }
+    it { should include("timestamp" => "12/May/2014:07:54:01 +0000" ) }
+
+    it { should include("clientip" => "10.0.1.2" ) unless ecs_compatibility? }
+    it { should include("requester" => "-" ) unless ecs_compatibility? }
+    it { should include("client" => { 'ip' => '10.0.1.2' } ) if ecs_compatibility? }
+
+    it { should include("request_id" => "7ACC4BE89EXAMPLE" ) unless ecs_compatibility? }
+    it { should include("operation" => "REST.GET.OBJECT" ) unless ecs_compatibility? }
+
+    it do
+      if ecs_compatibility?
+        should include("aws"=>{"s3access"=>hash_including("key" => "foo/bar.html")})
+      else
+        should include("key" => "foo/bar.html")
+      end
+    end
+
+    it { should include("verb" => "GET" ) unless ecs_compatibility? }
+    it { should include("request" => "/foo/bar.html" ) unless ecs_compatibility? }
+    it { should include("httpversion" => "1.1" ) unless ecs_compatibility? }
+    it { should include("response" => 304 ) unless ecs_compatibility? }
+    it { should include("object_size" => 1718 ) unless ecs_compatibility? }
+
+    it { should include("request_time_ms" => 10 ) unless ecs_compatibility? }
+    it { should include("referrer" => "\"-\"" ) unless ecs_compatibility? }
+
+    it { should include("agent" => "\"Mozilla/5.0\"" ) unless ecs_compatibility? }
+    it { should include("user_agent"=>{"original"=>"Mozilla/5.0"}) if ecs_compatibility? }
+
+    ["tags", "error_code", "turnaround_time_ms", "version_id", "bytes"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil unless ecs_compatibility?
+      end
+    end
+
+  end
+
+  context 'a long line' do
+
+    let(:message) do
+      '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket1 [06/Feb/2019:00:00:38 +0000] ' +
+      '192.0.2.3 arn:aws:iam::123456:user/test@elastic.co A1206F460EXAMPLE REST.GET.BUCKETPOLICY - ' +
+      '"GET /awsexamplebucket1?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 12 "-" ' +
+      '"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64" - ' +
+      'BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 ' +
+      'AuthHeader awsexamplebucket1.s3.us-west-1.amazonaws.com TLSV1.1'
+    end
+
+    it 'matches' do
+      if ecs_compatibility?
+        expect(grok).to include("client"=>{"ip"=>"192.0.2.3", "user"=>{"id"=>"arn:aws:iam::123456:user/test@elastic.co"}})
+        expect(grok).to include("http"=>{"request"=>{"method"=>"GET"}, "version"=>"1.1", "response"=>{"status_code"=>404}})
+        expect(grok).to include("url"=>{"original"=>"/awsexamplebucket1?policy"})
+        expect(grok).to include("aws"=>{"s3access"=>{
+            "bucket_owner"=>"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+            "bucket"=>"awsexamplebucket1",
+            "request_id"=>"A1206F460EXAMPLE",
+            "operation"=>"REST.GET.BUCKETPOLICY",
+            "turn_around_time"=>12,
+            "bytes_sent"=>297,
+            "request_uri"=>"GET /awsexamplebucket1?policy HTTP/1.1", # NOTE: redundant (beats compatibility)
+            "error_code"=>"NoSuchBucketPolicy",
+            "total_time" => 38,
+            # these fields weren't matched in legacy mode:
+            # Host Id -> Signature Version -> Cipher Suite -> Authentication Type -> Host Header -> TLS version
+            "host_id" => "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=",
+            "signature_version" => "SigV2",
+            #"cipher_suite" => "ECDHE-RSA-AES128-GCM-SHA256", # tls.cipher
+            "authentication_type" => "AuthHeader",
+            "host_header" => "awsexamplebucket1.s3.us-west-1.amazonaws.com",
+            "tls_version" => "TLSV1.1"
+        }})
+        expect(grok).to include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+        expect(grok).to include("user_agent"=>{
+            "original"=>"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64"
+        })
+      else
+        expect(grok).to include("owner"=>"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+                                "bucket"=>"awsexamplebucket1",
+                                "timestamp"=>"06/Feb/2019:00:00:38 +0000",
+                                "clientip"=>"192.0.2.3",
+                                "requester"=>"arn:aws:iam::123456:user/test@elastic.co",
+                                "request_id"=>"A1206F460EXAMPLE",
+                                "operation"=>"REST.GET.BUCKETPOLICY",
+                                "key"=>"-",
+                                "verb"=>"GET",
+                                "request"=>"/awsexamplebucket1?policy",
+                                "httpversion"=>"1.1",
+                                "response"=>404,
+                                "error_code"=>"NoSuchBucketPolicy",
+                                "bytes"=>297,
+                                # object_size nil
+                                "request_time_ms"=>38,
+                                "turnaround_time_ms"=>12,
+                                "referrer"=>"\"-\"",
+                                "agent"=>"\"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64\"")
+      end
+    end
+
+  end
+end
+
+describe_pattern "CLOUDFRONT_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  let(:message) do
+    "2016-06-10	18:41:39	IAD53	224281	192.168.1.1	GET	d27enomp470abc.cloudfront.net	/content/sample/thing.pdf	200	https://example.com/	Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36	-	-	Miss	UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==	host.example.com	https	883	0.036	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	RefreshHit"
+  end
+
+  it 'matches' do
+    should include("timestamp" => "2016-06-10	18:41:39" )
+
+    if ecs_compatibility?
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_location"=>"IAD53")})
+      should include("destination"=>{"bytes"=>224281})
+      should include("source"=>{"ip"=>"192.168.1.1", "bytes"=>883})
+      should include("http"=>{
+          "request"=>{"method"=>"GET", "referrer"=>"https://example.com/"},
+          "response"=>{"status_code"=>200}
+      })
+      should include("user_agent"=>{"original"=>"Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36"})
+      should include("url"=>{"domain"=>"d27enomp470abc.cloudfront.net", "path"=>"/content/sample/thing.pdf"})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_result_type"=>"Miss")})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_request_id"=>'UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==')})
+      should include("network"=>{"protocol"=>"https"})
+      should include("aws"=>{"cloudfront"=>hash_including("http"=>{"request"=>{"host"=>"host.example.com"}})})
+      should include("aws"=>{"cloudfront"=>hash_including("time_taken"=>0.036)})
+      should include("aws"=>{"cloudfront"=>hash_including("ssl_protocol"=>"TLSv1.2")})
+      should include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_response_result_type"=>"RefreshHit")})
+    else
+      should include("x_edge_location" => "IAD53" )
+      should include("sc_bytes" => 224281 )
+      should include("clientip" => "192.168.1.1" )
+      should include("cs_method" =>  "GET" )
+      should include("cs_host" => "d27enomp470abc.cloudfront.net" )
+      should include("cs_uri_stem" => "/content/sample/thing.pdf" )
+      should include("sc_status" => 200 )
+      should include("referrer" => "https://example.com/" )
+      should include("agent" => "Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36" )
+      should include("cs_uri_query" => "-" )
+      should include("cookies" => "-" )
+      should include("x_edge_result_type" => "Miss" )
+      should include("x_edge_request_id" => "UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==" )
+      should include("x_host_header" => "host.example.com" )
+      should include("cs_protocol" => "https" )
+      should include("cs_bytes" => 883 )
+      should include("time_taken" => 0.036 )
+      should include("x_forwarded_for" => "-" )
+      should include("ssl_protocol" => "TLSv1.2" )
+      should include("ssl_cipher" => "ECDHE-RSA-AES128-GCM-SHA256" )
+      should include("x_edge_response_result_type" => "RefreshHit" )
+    end
+  end
+
+  ["tags", "params"].each do |attribute|
+    it "have #{attribute} as nil" do
+      expect(subject[attribute]).to be_nil
+    end
+  end
+
+  context 'version 1.0' do # more fields at the end
+
+    let(:message) do
+      # Version: 1.0  -  https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
+      # Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end
+      "2019-12-04	21:02:31	LAX1	392	192.0.2.100	GET	d111111abcdef8.cloudfront.net	/index.html	200	-	Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36	-	-	Hit	SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==	d111111abcdef8.cloudfront.net	https	23	0.001	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	Hit	HTTP/2.0	-	-	11040	0.001	Hit	text/html	78	-	-"
+    end
+
+    it 'matches' do
+      should include("timestamp" => "2019-12-04\t21:02:31")
+
+      if ecs_compatibility?
+        should include("destination"=>{"bytes"=>392}, "source"=>{"ip"=>"192.0.2.100", "bytes"=>23, "port"=>11040}) # source.port not matched in legacy mode
+        should include("url"=>{"domain"=>"d111111abcdef8.cloudfront.net", "path"=>"/index.html"})
+        should include("http"=>hash_including("request"=>{"mime_type"=>"text/html", "method"=>"GET"}, "response"=>{"status_code"=>200})) # mime_type not matched in legacy mode
+        should include("user_agent"=>{"original"=>"Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36"})
+        should include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+        should include("network"=>{"protocol"=>"https"})
+
+        should include("http"=>hash_including("version"=>"2.0"))
+        should include("aws"=>{"cloudfront"=>{
+            "x_edge_location"=>"LAX1",
+            "x_edge_request_id"=>"SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==", # event.id
+            "x_edge_result_type"=>"Hit",
+            "x_edge_response_result_type"=>"Hit",
+            "x_edge_detailed_result_type"=>"Hit", # not captured in legacy mode
+            "time_taken"=>0.001,
+            "time_to_first_byte"=>0.001, # not captured in legacy mode
+            "http"=>{"request"=>{"host"=>"d111111abcdef8.cloudfront.net", "size"=>78}}, # http.request.size not captured in legacy mode
+            "ssl_protocol"=>"TLSv1.2",
+        }})
+      else
+        should include("cs_method"=>"GET", "cs_host"=>"d111111abcdef8.cloudfront.net", "cs_uri_stem"=>"/index.html", "cs_protocol"=>"https", "cs_bytes"=>23)
+        should include("x_host_header"=>"d111111abcdef8.cloudfront.net")
+        should include("time_taken"=>0.001)
+        should include("x_edge_request_id"=>"SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==")
+        should include("agent"=>"Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36")
+
+        should include("x_forwarded_for"=>"text/html") # TODO the legacy pattern does not handle the long(er) format correctly
+      end
+    end
+
+  end
+
+end