logstash-patterns-core 4.0.2 → 4.3.0

data/patterns/legacy/httpd

@@ -0,0 +1,15 @@
+HTTPDUSER %{EMAILADDRESS}|%{USER}
+HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
+
+# Log formats
+HTTPD_COMMONLOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" (?:-|%{NUMBER:response}) (?:-|%{NUMBER:bytes})
+HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}
+
+# Error logs
+HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}
+HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[(?:%{WORD:module})?:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}
+HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
+
+# Deprecated
+COMMONAPACHELOG %{HTTPD_COMMONLOG}
+COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}

data/patterns/{java → legacy/java}

@@ -1,15 +1,12 @@
 JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
 #Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
-JAVAFILE (?:[A-Za-z0-9_. -]+)
+JAVAFILE (?:[a-zA-Z$_0-9. -]+)
 #Allow special <init>, <clinit> methods
 JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
 #Line number is optional in special cases 'Native method' or 'Unknown source'
 JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
 # Java Logs
 JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
-JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
-JAVAFILE (?:[A-Za-z0-9_.-]+)
-JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
 JAVALOGMESSAGE (.*)
 # MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
 CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)

data/patterns/{linux-syslog → legacy/linux-syslog}

@@ -11,6 +11,6 @@ SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
 # IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
 SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>
 SYSLOG5424SD \[%{DATA}\]+
-SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
+SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{IPORHOST:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
 
 SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}

data/patterns/legacy/maven

@@ -0,0 +1 @@
+MAVEN_VERSION (?:(\d+)\.)?(?:(\d+)\.)?(\*|\d+)(?:[.-](RELEASE|SNAPSHOT))?

data/patterns/{nagios → legacy/nagios}

@@ -89,7 +89,7 @@ NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:
 NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
 NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
 
-NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2}
+NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{NUMBER:nagios_unknown1};%{NUMBER:nagios_unknown2}
 
 ####################
 #### External checks

data/patterns/legacy/redis

@@ -0,0 +1,3 @@
+REDISTIMESTAMP %{MONTHDAY} %{MONTH} %{TIME}
+REDISLOG \[%{POSINT:pid}\] %{REDISTIMESTAMP:timestamp} \* 
+REDISMONLOG %{NUMBER:timestamp} \[%{INT:database} %{IP:client}:%{NUMBER:port}\] "%{WORD:command}"\s?%{GREEDYDATA:params}

data/patterns/legacy/squid

@@ -0,0 +1,4 @@
+# Pattern squid3
+# Documentation of squid3 logs formats can be found at the following link:
+# http://wiki.squid-cache.org/Features/LogFormat
+SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:duration}\s%{IP:client_address}\s%{WORD:cache_result}/%{NONNEGINT:status_code}\s%{NUMBER:bytes}\s%{WORD:request_method}\s%{NOTSPACE:url}\s(%{NOTSPACE:user}|-)\s%{WORD:hierarchy_code}/(%{IPORHOST:server}|-)\s%{NOTSPACE:content_type}

data/spec/patterns/aws_spec.rb

@@ -0,0 +1,395 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe_pattern "ELB_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  context "parsing an access log" do
+
+    let(:message) do
+      "2014-02-15T23:39:43.945958Z my-test-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\""
+    end
+
+    it 'matches' do
+      should include("timestamp" => "2014-02-15T23:39:43.945958Z" )
+      if ecs_compatibility?
+        expect(grok).to include("aws" => { "elb" => {
+            "name"=>"my-test-loadbalancer",
+            "request_processing_time"=>{"sec"=>0.000073},
+            "response_processing_time"=>{"sec"=>0.000057},
+            "backend_processing_time"=>{"sec"=>0.001048},
+            "backend"=>{
+                "ip"=>"10.0.0.1", "port"=>80,
+                "http"=>{"response"=>{"status_code"=>200}}
+            }
+        }})
+        expect(grok).to include("http"=>{
+            "request"=>{"body"=>{"bytes"=>0}, "method"=>"GET"},
+            "response"=>{"body"=>{"bytes"=>29}, "status_code"=>200},
+            "version"=>"1.1"
+        })
+        expect(grok).to include("source"=>{"ip"=>"192.168.131.39", "port"=>2817})
+        expect(grok).to include("url"=>{
+            "original"=>"http://www.example.com:80/",
+            "port"=>80, "path"=>"/", "domain"=>"www.example.com", "scheme"=>"http"
+        })
+      else
+        should include("elb" => "my-test-loadbalancer" )
+        should include("clientip" => "192.168.131.39" )
+        should include("clientport" => 2817 )
+        should include("backendip" => "10.0.0.1" )
+        should include("backendport" => 80 )
+        should include("request_processing_time" => 0.000073 )
+        should include("backend_processing_time" => 0.001048 )
+        should include("response_processing_time" => 0.000057 )
+        should include("response" => 200 )
+        should include("backend_response" => 200 )
+        should include("received_bytes" => 0 )
+        should include("bytes" => 29 )
+        should include("verb" => "GET" )
+        should include("request" => "http://www.example.com:80/" )
+        should include("proto" => "http" )
+        should include("httpversion" => "1.1" )
+        should include("urihost" => "www.example.com:80" )
+        should include("path" => "/" )
+      end
+    end
+
+    ["tags", "params"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil
+      end
+    end
+  end
+
+  context "parsing a PUT request access log with missing backend info" do
+
+    let(:message) do
+      '2015-04-10T08:11:09.865823Z us-west-1-production-media 49.150.87.133:55128 - -1 -1 -1 408 - 1294336 0 "PUT https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw HTTP/1.1"'
+    end
+
+    it "matches" do
+      expect(grok).to include("timestamp"=>"2015-04-10T08:11:09.865823Z")
+      if ecs_compatibility?
+        expect(grok).to include("url"=>{
+            "original"=>"https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw",
+            "scheme"=>"https", "port"=>443, "path"=>"/videos/F4_M-T4X0MM6Hvy1PFHesw", "domain"=>"media.xxxyyyzzz.com"
+        })
+        expect(grok).to include("source"=>{"port"=>55128, "ip"=>"49.150.87.133"})
+        expect(grok).to include("http"=>{
+            "request"=>{"method"=>"PUT", "body"=>{"bytes"=>1294336}}, "version"=>"1.1",
+            "response"=>{"body"=>{"bytes"=>0}, "status_code"=>408}
+        })
+        # no backend.ip and backend.port
+        # no backend.http.status.code
+        # no request_processing_time.sec and friends
+        expect(grok).to include("aws"=>{"elb"=>{"name"=>"us-west-1-production-media"}})
+      else
+        expect(grok).to include(
+            "elb"=>"us-west-1-production-media",
+            "clientip"=>"49.150.87.133", "clientport"=>55128,
+            "response_processing_time"=>-1.0,
+            "request_processing_time"=>-1.0,
+            "backend_processing_time"=>-1.0,
+            "response"=>408,
+            "received_bytes"=>1294336,
+            "bytes"=>0,
+            "verb"=>"PUT",
+            "request"=>"https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw",
+            "port"=>"443", "proto"=>"https", "path"=>"/videos/F4_M-T4X0MM6Hvy1PFHesw", "urihost"=>"media.xxxyyyzzz.com:443",
+            "httpversion"=>"1.1")
+
+        expect(grok.keys).to_not include("backendip", "backendport", "backendresponse")
+      end
+    end
+
+  end
+
+  context '(new) https format' do # slightly longer - 3 fields added at the end
+
+    let(:message) do
+      '2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.38.0 (#56-0ef1d4a5)" DHE-RSA-AES128-SHA TLSv1.2'
+    end
+
+    it 'matches (new) suffix fields' do
+      if ecs_compatibility?
+        expect(grok).to include "tls" => { "cipher" => "DHE-RSA-AES128-SHA" }
+        expect(grok).to include "aws" => { "elb" => hash_including("ssl_protocol" => 'TLSv1.2')}
+        expect(grok).to include "user_agent"=>{"original"=>"curl/7.38.0 (#56-0ef1d4a5)"}
+      end
+    end
+
+    context 'with optional fields' do
+
+      let(:message) do
+        '2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "-" - -'
+      end
+
+      it 'matches (new) suffix fields' do
+        if ecs_compatibility?
+          expect(grok.keys).to_not include "tls"
+          expect(grok['aws']['elb'].keys).to_not include "ssl_protocol"
+          expect(grok.keys).to_not include "user_agent"
+        end
+      end
+
+    end
+
+  end
+end
+
+describe_pattern "S3_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  context "parsing GET.VERSIONING message" do
+
+    let(:message) do
+      "79a5 mybucket [06/Feb/2014:00:00:38 +0000] 192.0.2.3 79a5 3E57427F3EXAMPLE REST.GET.VERSIONING - \"GET /mybucket?versioning HTTP/1.1\" 200 - 113 - 7 - \"-\" \"S3Console/0.4\" -"
+    end
+
+    it { should include("owner" => "79a5" ) unless ecs_compatibility? }
+    it { should include("bucket" => "mybucket" ) unless ecs_compatibility? }
+    it { should include("timestamp" => "06/Feb/2014:00:00:38 +0000" ) unless ecs_compatibility? }
+    it { should include("clientip" => "192.0.2.3" ) unless ecs_compatibility? }
+    it { should include("requester" => "79a5" ) unless ecs_compatibility? }
+    it { should include("request_id" => "3E57427F3EXAMPLE" ) unless ecs_compatibility? }
+    it { should include("operation" => "REST.GET.VERSIONING" ) unless ecs_compatibility? }
+    it { should include("key" => "-" ) unless ecs_compatibility? }
+
+    it { should include("verb" => "GET" ) unless ecs_compatibility? }
+    it { should include("request" => "/mybucket?versioning" ) unless ecs_compatibility? }
+    it { should include("httpversion" => "1.1" ) unless ecs_compatibility? }
+    it { should include("response" => 200 ) unless ecs_compatibility? }
+    it { should include("bytes" => 113 ) unless ecs_compatibility? }
+
+    it { should include("request_time_ms" => 7 ) unless ecs_compatibility? }
+    it { should include("referrer" => "\"-\"" ) unless ecs_compatibility? }
+    it { should include("agent" => "\"S3Console/0.4\"" ) unless ecs_compatibility? }
+
+    ["tags", "error_code", "object_size", "turnaround_time_ms", "version_id"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil unless ecs_compatibility?
+      end
+    end
+
+  end
+
+  context "parsing a GET.OBJECT message" do
+
+    let(:message) do
+      "79a5 mybucket [12/May/2014:07:54:01 +0000] 10.0.1.2 - 7ACC4BE89EXAMPLE REST.GET.OBJECT foo/bar.html \"GET /foo/bar.html HTTP/1.1\" 304 - - 1718 10 - \"-\" \"Mozilla/5.0\" -"
+    end
+
+    it do
+      if ecs_compatibility?
+        should include("aws"=>{"s3access"=>hash_including("bucket_owner" => "79a5")})
+      else
+        should include("owner" => "79a5")
+      end
+    end
+
+    it { should include("bucket" => "mybucket" ) unless ecs_compatibility? }
+    it { should include("timestamp" => "12/May/2014:07:54:01 +0000" ) }
+
+    it { should include("clientip" => "10.0.1.2" ) unless ecs_compatibility? }
+    it { should include("requester" => "-" ) unless ecs_compatibility? }
+    it { should include("client" => { 'ip' => '10.0.1.2' } ) if ecs_compatibility? }
+
+    it { should include("request_id" => "7ACC4BE89EXAMPLE" ) unless ecs_compatibility? }
+    it { should include("operation" => "REST.GET.OBJECT" ) unless ecs_compatibility? }
+
+    it do
+      if ecs_compatibility?
+        should include("aws"=>{"s3access"=>hash_including("key" => "foo/bar.html")})
+      else
+        should include("key" => "foo/bar.html")
+      end
+    end
+
+    it { should include("verb" => "GET" ) unless ecs_compatibility? }
+    it { should include("request" => "/foo/bar.html" ) unless ecs_compatibility? }
+    it { should include("httpversion" => "1.1" ) unless ecs_compatibility? }
+    it { should include("response" => 304 ) unless ecs_compatibility? }
+    it { should include("object_size" => 1718 ) unless ecs_compatibility? }
+
+    it { should include("request_time_ms" => 10 ) unless ecs_compatibility? }
+    it { should include("referrer" => "\"-\"" ) unless ecs_compatibility? }
+
+    it { should include("agent" => "\"Mozilla/5.0\"" ) unless ecs_compatibility? }
+    it { should include("user_agent"=>{"original"=>"Mozilla/5.0"}) if ecs_compatibility? }
+
+    ["tags", "error_code", "turnaround_time_ms", "version_id", "bytes"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil unless ecs_compatibility?
+      end
+    end
+
+  end
+
+  context 'a long line' do
+
+    let(:message) do
+      '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket1 [06/Feb/2019:00:00:38 +0000] ' +
+      '192.0.2.3 arn:aws:iam::123456:user/test@elastic.co A1206F460EXAMPLE REST.GET.BUCKETPOLICY - ' +
+      '"GET /awsexamplebucket1?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 12 "-" ' +
+      '"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64" - ' +
+      'BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 ' +
+      'AuthHeader awsexamplebucket1.s3.us-west-1.amazonaws.com TLSV1.1'
+    end
+
+    it 'matches' do
+      if ecs_compatibility?
+        expect(grok).to include("client"=>{"ip"=>"192.0.2.3", "user"=>{"id"=>"arn:aws:iam::123456:user/test@elastic.co"}})
+        expect(grok).to include("http"=>{"request"=>{"method"=>"GET"}, "version"=>"1.1", "response"=>{"status_code"=>404}})
+        expect(grok).to include("url"=>{"original"=>"/awsexamplebucket1?policy"})
+        expect(grok).to include("aws"=>{"s3access"=>{
+            "bucket_owner"=>"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+            "bucket"=>"awsexamplebucket1",
+            "request_id"=>"A1206F460EXAMPLE",
+            "operation"=>"REST.GET.BUCKETPOLICY",
+            "turn_around_time"=>12,
+            "bytes_sent"=>297,
+            "request_uri"=>"GET /awsexamplebucket1?policy HTTP/1.1", # NOTE: redundant (beats compatibility)
+            "error_code"=>"NoSuchBucketPolicy",
+            "total_time" => 38,
+            # these fields weren't matched in legacy mode:
+            # Host Id -> Signature Version -> Cipher Suite -> Authentication Type -> Host Header -> TLS version
+            "host_id" => "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=",
+            "signature_version" => "SigV2",
+            #"cipher_suite" => "ECDHE-RSA-AES128-GCM-SHA256", # tls.cipher
+            "authentication_type" => "AuthHeader",
+            "host_header" => "awsexamplebucket1.s3.us-west-1.amazonaws.com",
+            "tls_version" => "TLSV1.1"
+        }})
+        expect(grok).to include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+        expect(grok).to include("user_agent"=>{
+            "original"=>"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64"
+        })
+      else
+        expect(grok).to include("owner"=>"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+                                "bucket"=>"awsexamplebucket1",
+                                "timestamp"=>"06/Feb/2019:00:00:38 +0000",
+                                "clientip"=>"192.0.2.3",
+                                "requester"=>"arn:aws:iam::123456:user/test@elastic.co",
+                                "request_id"=>"A1206F460EXAMPLE",
+                                "operation"=>"REST.GET.BUCKETPOLICY",
+                                "key"=>"-",
+                                "verb"=>"GET",
+                                "request"=>"/awsexamplebucket1?policy",
+                                "httpversion"=>"1.1",
+                                "response"=>404,
+                                "error_code"=>"NoSuchBucketPolicy",
+                                "bytes"=>297,
+                                # object_size nil
+                                "request_time_ms"=>38,
+                                "turnaround_time_ms"=>12,
+                                "referrer"=>"\"-\"",
+                                "agent"=>"\"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64\"")
+      end
+    end
+
+  end
+end
+
+describe_pattern "CLOUDFRONT_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  let(:message) do
+    "2016-06-10	18:41:39	IAD53	224281	192.168.1.1	GET	d27enomp470abc.cloudfront.net	/content/sample/thing.pdf	200	https://example.com/	Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36	-	-	Miss	UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==	host.example.com	https	883	0.036	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	RefreshHit"
+  end
+
+  it 'matches' do
+    should include("timestamp" => "2016-06-10	18:41:39" )
+
+    if ecs_compatibility?
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_location"=>"IAD53")})
+      should include("destination"=>{"bytes"=>224281})
+      should include("source"=>{"ip"=>"192.168.1.1", "bytes"=>883})
+      should include("http"=>{
+          "request"=>{"method"=>"GET", "referrer"=>"https://example.com/"},
+          "response"=>{"status_code"=>200}
+      })
+      should include("user_agent"=>{"original"=>"Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36"})
+      should include("url"=>{"domain"=>"d27enomp470abc.cloudfront.net", "path"=>"/content/sample/thing.pdf"})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_result_type"=>"Miss")})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_request_id"=>'UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==')})
+      should include("network"=>{"protocol"=>"https"})
+      should include("aws"=>{"cloudfront"=>hash_including("http"=>{"request"=>{"host"=>"host.example.com"}})})
+      should include("aws"=>{"cloudfront"=>hash_including("time_taken"=>0.036)})
+      should include("aws"=>{"cloudfront"=>hash_including("ssl_protocol"=>"TLSv1.2")})
+      should include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_response_result_type"=>"RefreshHit")})
+    else
+      should include("x_edge_location" => "IAD53" )
+      should include("sc_bytes" => 224281 )
+      should include("clientip" => "192.168.1.1" )
+      should include("cs_method" =>  "GET" )
+      should include("cs_host" => "d27enomp470abc.cloudfront.net" )
+      should include("cs_uri_stem" => "/content/sample/thing.pdf" )
+      should include("sc_status" => 200 )
+      should include("referrer" => "https://example.com/" )
+      should include("agent" => "Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36" )
+      should include("cs_uri_query" => "-" )
+      should include("cookies" => "-" )
+      should include("x_edge_result_type" => "Miss" )
+      should include("x_edge_request_id" => "UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==" )
+      should include("x_host_header" => "host.example.com" )
+      should include("cs_protocol" => "https" )
+      should include("cs_bytes" => 883 )
+      should include("time_taken" => 0.036 )
+      should include("x_forwarded_for" => "-" )
+      should include("ssl_protocol" => "TLSv1.2" )
+      should include("ssl_cipher" => "ECDHE-RSA-AES128-GCM-SHA256" )
+      should include("x_edge_response_result_type" => "RefreshHit" )
+    end
+  end
+
+  ["tags", "params"].each do |attribute|
+    it "have #{attribute} as nil" do
+      expect(subject[attribute]).to be_nil
+    end
+  end
+
+  context 'version 1.0' do # more fields at the end
+
+    let(:message) do
+      # Version: 1.0  -  https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
+      # Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end
+      "2019-12-04	21:02:31	LAX1	392	192.0.2.100	GET	d111111abcdef8.cloudfront.net	/index.html	200	-	Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36	-	-	Hit	SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==	d111111abcdef8.cloudfront.net	https	23	0.001	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	Hit	HTTP/2.0	-	-	11040	0.001	Hit	text/html	78	-	-"
+    end
+
+    it 'matches' do
+      should include("timestamp" => "2019-12-04\t21:02:31")
+
+      if ecs_compatibility?
+        should include("destination"=>{"bytes"=>392}, "source"=>{"ip"=>"192.0.2.100", "bytes"=>23, "port"=>11040}) # source.port not matched in legacy mode
+        should include("url"=>{"domain"=>"d111111abcdef8.cloudfront.net", "path"=>"/index.html"})
+        should include("http"=>hash_including("request"=>{"mime_type"=>"text/html", "method"=>"GET"}, "response"=>{"status_code"=>200})) # mime_type not matched in legacy mode
+        should include("user_agent"=>{"original"=>"Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36"})
+        should include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+        should include("network"=>{"protocol"=>"https"})
+
+        should include("http"=>hash_including("version"=>"2.0"))
+        should include("aws"=>{"cloudfront"=>{
+            "x_edge_location"=>"LAX1",
+            "x_edge_request_id"=>"SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==", # event.id
+            "x_edge_result_type"=>"Hit",
+            "x_edge_response_result_type"=>"Hit",
+            "x_edge_detailed_result_type"=>"Hit", # not captured in legacy mode
+            "time_taken"=>0.001,
+            "time_to_first_byte"=>0.001, # not captured in legacy mode
+            "http"=>{"request"=>{"host"=>"d111111abcdef8.cloudfront.net", "size"=>78}}, # http.request.size not captured in legacy mode
+            "ssl_protocol"=>"TLSv1.2",
+        }})
+      else
+        should include("cs_method"=>"GET", "cs_host"=>"d111111abcdef8.cloudfront.net", "cs_uri_stem"=>"/index.html", "cs_protocol"=>"https", "cs_bytes"=>23)
+        should include("x_host_header"=>"d111111abcdef8.cloudfront.net")
+        should include("time_taken"=>0.001)
+        should include("x_edge_request_id"=>"SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==")
+        should include("agent"=>"Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36")
+
+        should include("x_forwarded_for"=>"text/html") # TODO the legacy pattern does not handle the long(er) format correctly
+      end
+    end
+
+  end
+
+end