logstash-patterns-core 4.1.0 → 4.3.1

data/patterns/{nagios → legacy/nagios}

@@ -89,7 +89,7 @@ NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:
 NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
 NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
 
-NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2}
+NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{NUMBER:nagios_unknown1};%{NUMBER:nagios_unknown2}
 
 ####################
 #### External checks

data/patterns/legacy/squid

@@ -0,0 +1,4 @@
+# Pattern squid3
+# Documentation of squid3 logs formats can be found at the following link:
+# http://wiki.squid-cache.org/Features/LogFormat
+SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:duration}\s%{IP:client_address}\s%{WORD:cache_result}/%{NONNEGINT:status_code}\s%{NUMBER:bytes}\s%{WORD:request_method}\s%{NOTSPACE:url}\s(%{NOTSPACE:user}|-)\s%{WORD:hierarchy_code}/(%{IPORHOST:server}|-)\s%{NOTSPACE:content_type}

data/spec/patterns/aws_spec.rb

@@ -0,0 +1,395 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe_pattern "ELB_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  context "parsing an access log" do
+
+    let(:message) do
+      "2014-02-15T23:39:43.945958Z my-test-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\""
+    end
+
+    it 'matches' do
+      should include("timestamp" => "2014-02-15T23:39:43.945958Z" )
+      if ecs_compatibility?
+        expect(grok).to include("aws" => { "elb" => {
+            "name"=>"my-test-loadbalancer",
+            "request_processing_time"=>{"sec"=>0.000073},
+            "response_processing_time"=>{"sec"=>0.000057},
+            "backend_processing_time"=>{"sec"=>0.001048},
+            "backend"=>{
+                "ip"=>"10.0.0.1", "port"=>80,
+                "http"=>{"response"=>{"status_code"=>200}}
+            }
+        }})
+        expect(grok).to include("http"=>{
+            "request"=>{"body"=>{"bytes"=>0}, "method"=>"GET"},
+            "response"=>{"body"=>{"bytes"=>29}, "status_code"=>200},
+            "version"=>"1.1"
+        })
+        expect(grok).to include("source"=>{"ip"=>"192.168.131.39", "port"=>2817})
+        expect(grok).to include("url"=>{
+            "original"=>"http://www.example.com:80/",
+            "port"=>80, "path"=>"/", "domain"=>"www.example.com", "scheme"=>"http"
+        })
+      else
+        should include("elb" => "my-test-loadbalancer" )
+        should include("clientip" => "192.168.131.39" )
+        should include("clientport" => 2817 )
+        should include("backendip" => "10.0.0.1" )
+        should include("backendport" => 80 )
+        should include("request_processing_time" => 0.000073 )
+        should include("backend_processing_time" => 0.001048 )
+        should include("response_processing_time" => 0.000057 )
+        should include("response" => 200 )
+        should include("backend_response" => 200 )
+        should include("received_bytes" => 0 )
+        should include("bytes" => 29 )
+        should include("verb" => "GET" )
+        should include("request" => "http://www.example.com:80/" )
+        should include("proto" => "http" )
+        should include("httpversion" => "1.1" )
+        should include("urihost" => "www.example.com:80" )
+        should include("path" => "/" )
+      end
+    end
+
+    ["tags", "params"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil
+      end
+    end
+  end
+
+  context "parsing a PUT request access log with missing backend info" do
+
+    let(:message) do
+      '2015-04-10T08:11:09.865823Z us-west-1-production-media 49.150.87.133:55128 - -1 -1 -1 408 - 1294336 0 "PUT https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw HTTP/1.1"'
+    end
+
+    it "matches" do
+      expect(grok).to include("timestamp"=>"2015-04-10T08:11:09.865823Z")
+      if ecs_compatibility?
+        expect(grok).to include("url"=>{
+            "original"=>"https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw",
+            "scheme"=>"https", "port"=>443, "path"=>"/videos/F4_M-T4X0MM6Hvy1PFHesw", "domain"=>"media.xxxyyyzzz.com"
+        })
+        expect(grok).to include("source"=>{"port"=>55128, "ip"=>"49.150.87.133"})
+        expect(grok).to include("http"=>{
+            "request"=>{"method"=>"PUT", "body"=>{"bytes"=>1294336}}, "version"=>"1.1",
+            "response"=>{"body"=>{"bytes"=>0}, "status_code"=>408}
+        })
+        # no backend.ip and backend.port
+        # no backend.http.status.code
+        # no request_processing_time.sec and friends
+        expect(grok).to include("aws"=>{"elb"=>{"name"=>"us-west-1-production-media"}})
+      else
+        expect(grok).to include(
+            "elb"=>"us-west-1-production-media",
+            "clientip"=>"49.150.87.133", "clientport"=>55128,
+            "response_processing_time"=>-1.0,
+            "request_processing_time"=>-1.0,
+            "backend_processing_time"=>-1.0,
+            "response"=>408,
+            "received_bytes"=>1294336,
+            "bytes"=>0,
+            "verb"=>"PUT",
+            "request"=>"https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw",
+            "port"=>"443", "proto"=>"https", "path"=>"/videos/F4_M-T4X0MM6Hvy1PFHesw", "urihost"=>"media.xxxyyyzzz.com:443",
+            "httpversion"=>"1.1")
+
+        expect(grok.keys).to_not include("backendip", "backendport", "backendresponse")
+      end
+    end
+
+  end
+
+  context '(new) https format' do # slightly longer - 3 fields added at the end
+
+    let(:message) do
+      '2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.38.0 (#56-0ef1d4a5)" DHE-RSA-AES128-SHA TLSv1.2'
+    end
+
+    it 'matches (new) suffix fields' do
+      if ecs_compatibility?
+        expect(grok).to include "tls" => { "cipher" => "DHE-RSA-AES128-SHA" }
+        expect(grok).to include "aws" => { "elb" => hash_including("ssl_protocol" => 'TLSv1.2')}
+        expect(grok).to include "user_agent"=>{"original"=>"curl/7.38.0 (#56-0ef1d4a5)"}
+      end
+    end
+
+    context 'with optional fields' do
+
+      let(:message) do
+        '2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "-" - -'
+      end
+
+      it 'matches (new) suffix fields' do
+        if ecs_compatibility?
+          expect(grok.keys).to_not include "tls"
+          expect(grok['aws']['elb'].keys).to_not include "ssl_protocol"
+          expect(grok.keys).to_not include "user_agent"
+        end
+      end
+
+    end
+
+  end
+end
+
+describe_pattern "S3_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  context "parsing GET.VERSIONING message" do
+
+    let(:message) do
+      "79a5 mybucket [06/Feb/2014:00:00:38 +0000] 192.0.2.3 79a5 3E57427F3EXAMPLE REST.GET.VERSIONING - \"GET /mybucket?versioning HTTP/1.1\" 200 - 113 - 7 - \"-\" \"S3Console/0.4\" -"
+    end
+
+    it { should include("owner" => "79a5" ) unless ecs_compatibility? }
+    it { should include("bucket" => "mybucket" ) unless ecs_compatibility? }
+    it { should include("timestamp" => "06/Feb/2014:00:00:38 +0000" ) unless ecs_compatibility? }
+    it { should include("clientip" => "192.0.2.3" ) unless ecs_compatibility? }
+    it { should include("requester" => "79a5" ) unless ecs_compatibility? }
+    it { should include("request_id" => "3E57427F3EXAMPLE" ) unless ecs_compatibility? }
+    it { should include("operation" => "REST.GET.VERSIONING" ) unless ecs_compatibility? }
+    it { should include("key" => "-" ) unless ecs_compatibility? }
+
+    it { should include("verb" => "GET" ) unless ecs_compatibility? }
+    it { should include("request" => "/mybucket?versioning" ) unless ecs_compatibility? }
+    it { should include("httpversion" => "1.1" ) unless ecs_compatibility? }
+    it { should include("response" => 200 ) unless ecs_compatibility? }
+    it { should include("bytes" => 113 ) unless ecs_compatibility? }
+
+    it { should include("request_time_ms" => 7 ) unless ecs_compatibility? }
+    it { should include("referrer" => "\"-\"" ) unless ecs_compatibility? }
+    it { should include("agent" => "\"S3Console/0.4\"" ) unless ecs_compatibility? }
+
+    ["tags", "error_code", "object_size", "turnaround_time_ms", "version_id"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil unless ecs_compatibility?
+      end
+    end
+
+  end
+
+  context "parsing a GET.OBJECT message" do
+
+    let(:message) do
+      "79a5 mybucket [12/May/2014:07:54:01 +0000] 10.0.1.2 - 7ACC4BE89EXAMPLE REST.GET.OBJECT foo/bar.html \"GET /foo/bar.html HTTP/1.1\" 304 - - 1718 10 - \"-\" \"Mozilla/5.0\" -"
+    end
+
+    it do
+      if ecs_compatibility?
+        should include("aws"=>{"s3access"=>hash_including("bucket_owner" => "79a5")})
+      else
+        should include("owner" => "79a5")
+      end
+    end
+
+    it { should include("bucket" => "mybucket" ) unless ecs_compatibility? }
+    it { should include("timestamp" => "12/May/2014:07:54:01 +0000" ) }
+
+    it { should include("clientip" => "10.0.1.2" ) unless ecs_compatibility? }
+    it { should include("requester" => "-" ) unless ecs_compatibility? }
+    it { should include("client" => { 'ip' => '10.0.1.2' } ) if ecs_compatibility? }
+
+    it { should include("request_id" => "7ACC4BE89EXAMPLE" ) unless ecs_compatibility? }
+    it { should include("operation" => "REST.GET.OBJECT" ) unless ecs_compatibility? }
+
+    it do
+      if ecs_compatibility?
+        should include("aws"=>{"s3access"=>hash_including("key" => "foo/bar.html")})
+      else
+        should include("key" => "foo/bar.html")
+      end
+    end
+
+    it { should include("verb" => "GET" ) unless ecs_compatibility? }
+    it { should include("request" => "/foo/bar.html" ) unless ecs_compatibility? }
+    it { should include("httpversion" => "1.1" ) unless ecs_compatibility? }
+    it { should include("response" => 304 ) unless ecs_compatibility? }
+    it { should include("object_size" => 1718 ) unless ecs_compatibility? }
+
+    it { should include("request_time_ms" => 10 ) unless ecs_compatibility? }
+    it { should include("referrer" => "\"-\"" ) unless ecs_compatibility? }
+
+    it { should include("agent" => "\"Mozilla/5.0\"" ) unless ecs_compatibility? }
+    it { should include("user_agent"=>{"original"=>"Mozilla/5.0"}) if ecs_compatibility? }
+
+    ["tags", "error_code", "turnaround_time_ms", "version_id", "bytes"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil unless ecs_compatibility?
+      end
+    end
+
+  end
+
+  context 'a long line' do
+
+    let(:message) do
+      '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket1 [06/Feb/2019:00:00:38 +0000] ' +
+      '192.0.2.3 arn:aws:iam::123456:user/test@elastic.co A1206F460EXAMPLE REST.GET.BUCKETPOLICY - ' +
+      '"GET /awsexamplebucket1?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 12 "-" ' +
+      '"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64" - ' +
+      'BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 ' +
+      'AuthHeader awsexamplebucket1.s3.us-west-1.amazonaws.com TLSV1.1'
+    end
+
+    it 'matches' do
+      if ecs_compatibility?
+        expect(grok).to include("client"=>{"ip"=>"192.0.2.3", "user"=>{"id"=>"arn:aws:iam::123456:user/test@elastic.co"}})
+        expect(grok).to include("http"=>{"request"=>{"method"=>"GET"}, "version"=>"1.1", "response"=>{"status_code"=>404}})
+        expect(grok).to include("url"=>{"original"=>"/awsexamplebucket1?policy"})
+        expect(grok).to include("aws"=>{"s3access"=>{
+            "bucket_owner"=>"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+            "bucket"=>"awsexamplebucket1",
+            "request_id"=>"A1206F460EXAMPLE",
+            "operation"=>"REST.GET.BUCKETPOLICY",
+            "turn_around_time"=>12,
+            "bytes_sent"=>297,
+            "request_uri"=>"GET /awsexamplebucket1?policy HTTP/1.1", # NOTE: redundant (beats compatibility)
+            "error_code"=>"NoSuchBucketPolicy",
+            "total_time" => 38,
+            # these fields weren't matched in legacy mode:
+            # Host Id -> Signature Version -> Cipher Suite -> Authentication Type -> Host Header -> TLS version
+            "host_id" => "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=",
+            "signature_version" => "SigV2",
+            #"cipher_suite" => "ECDHE-RSA-AES128-GCM-SHA256", # tls.cipher
+            "authentication_type" => "AuthHeader",
+            "host_header" => "awsexamplebucket1.s3.us-west-1.amazonaws.com",
+            "tls_version" => "TLSV1.1"
+        }})
+        expect(grok).to include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+        expect(grok).to include("user_agent"=>{
+            "original"=>"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64"
+        })
+      else
+        expect(grok).to include("owner"=>"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+                                "bucket"=>"awsexamplebucket1",
+                                "timestamp"=>"06/Feb/2019:00:00:38 +0000",
+                                "clientip"=>"192.0.2.3",
+                                "requester"=>"arn:aws:iam::123456:user/test@elastic.co",
+                                "request_id"=>"A1206F460EXAMPLE",
+                                "operation"=>"REST.GET.BUCKETPOLICY",
+                                "key"=>"-",
+                                "verb"=>"GET",
+                                "request"=>"/awsexamplebucket1?policy",
+                                "httpversion"=>"1.1",
+                                "response"=>404,
+                                "error_code"=>"NoSuchBucketPolicy",
+                                "bytes"=>297,
+                                # object_size nil
+                                "request_time_ms"=>38,
+                                "turnaround_time_ms"=>12,
+                                "referrer"=>"\"-\"",
+                                "agent"=>"\"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64\"")
+      end
+    end
+
+  end
+end
+
+describe_pattern "CLOUDFRONT_ACCESS_LOG", ['legacy', 'ecs-v1'] do
+
+  let(:message) do
+    "2016-06-10	18:41:39	IAD53	224281	192.168.1.1	GET	d27enomp470abc.cloudfront.net	/content/sample/thing.pdf	200	https://example.com/	Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36	-	-	Miss	UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==	host.example.com	https	883	0.036	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	RefreshHit"
+  end
+
+  it 'matches' do
+    should include("timestamp" => "2016-06-10	18:41:39" )
+
+    if ecs_compatibility?
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_location"=>"IAD53")})
+      should include("destination"=>{"bytes"=>224281})
+      should include("source"=>{"ip"=>"192.168.1.1", "bytes"=>883})
+      should include("http"=>{
+          "request"=>{"method"=>"GET", "referrer"=>"https://example.com/"},
+          "response"=>{"status_code"=>200}
+      })
+      should include("user_agent"=>{"original"=>"Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36"})
+      should include("url"=>{"domain"=>"d27enomp470abc.cloudfront.net", "path"=>"/content/sample/thing.pdf"})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_result_type"=>"Miss")})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_request_id"=>'UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==')})
+      should include("network"=>{"protocol"=>"https"})
+      should include("aws"=>{"cloudfront"=>hash_including("http"=>{"request"=>{"host"=>"host.example.com"}})})
+      should include("aws"=>{"cloudfront"=>hash_including("time_taken"=>0.036)})
+      should include("aws"=>{"cloudfront"=>hash_including("ssl_protocol"=>"TLSv1.2")})
+      should include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+      should include("aws"=>{"cloudfront"=>hash_including("x_edge_response_result_type"=>"RefreshHit")})
+    else
+      should include("x_edge_location" => "IAD53" )
+      should include("sc_bytes" => 224281 )
+      should include("clientip" => "192.168.1.1" )
+      should include("cs_method" =>  "GET" )
+      should include("cs_host" => "d27enomp470abc.cloudfront.net" )
+      should include("cs_uri_stem" => "/content/sample/thing.pdf" )
+      should include("sc_status" => 200 )
+      should include("referrer" => "https://example.com/" )
+      should include("agent" => "Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36" )
+      should include("cs_uri_query" => "-" )
+      should include("cookies" => "-" )
+      should include("x_edge_result_type" => "Miss" )
+      should include("x_edge_request_id" => "UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==" )
+      should include("x_host_header" => "host.example.com" )
+      should include("cs_protocol" => "https" )
+      should include("cs_bytes" => 883 )
+      should include("time_taken" => 0.036 )
+      should include("x_forwarded_for" => "-" )
+      should include("ssl_protocol" => "TLSv1.2" )
+      should include("ssl_cipher" => "ECDHE-RSA-AES128-GCM-SHA256" )
+      should include("x_edge_response_result_type" => "RefreshHit" )
+    end
+  end
+
+  ["tags", "params"].each do |attribute|
+    it "have #{attribute} as nil" do
+      expect(subject[attribute]).to be_nil
+    end
+  end
+
+  context 'version 1.0' do # more fields at the end
+
+    let(:message) do
+      # Version: 1.0  -  https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
+      # Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end
+      "2019-12-04	21:02:31	LAX1	392	192.0.2.100	GET	d111111abcdef8.cloudfront.net	/index.html	200	-	Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36	-	-	Hit	SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==	d111111abcdef8.cloudfront.net	https	23	0.001	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	Hit	HTTP/2.0	-	-	11040	0.001	Hit	text/html	78	-	-"
+    end
+
+    it 'matches' do
+      should include("timestamp" => "2019-12-04\t21:02:31")
+
+      if ecs_compatibility?
+        should include("destination"=>{"bytes"=>392}, "source"=>{"ip"=>"192.0.2.100", "bytes"=>23, "port"=>11040}) # source.port not matched in legacy mode
+        should include("url"=>{"domain"=>"d111111abcdef8.cloudfront.net", "path"=>"/index.html"})
+        should include("http"=>hash_including("request"=>{"mime_type"=>"text/html", "method"=>"GET"}, "response"=>{"status_code"=>200})) # mime_type not matched in legacy mode
+        should include("user_agent"=>{"original"=>"Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36"})
+        should include("tls"=>{"cipher"=>"ECDHE-RSA-AES128-GCM-SHA256"})
+        should include("network"=>{"protocol"=>"https"})
+
+        should include("http"=>hash_including("version"=>"2.0"))
+        should include("aws"=>{"cloudfront"=>{
+            "x_edge_location"=>"LAX1",
+            "x_edge_request_id"=>"SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==", # event.id
+            "x_edge_result_type"=>"Hit",
+            "x_edge_response_result_type"=>"Hit",
+            "x_edge_detailed_result_type"=>"Hit", # not captured in legacy mode
+            "time_taken"=>0.001,
+            "time_to_first_byte"=>0.001, # not captured in legacy mode
+            "http"=>{"request"=>{"host"=>"d111111abcdef8.cloudfront.net", "size"=>78}}, # http.request.size not captured in legacy mode
+            "ssl_protocol"=>"TLSv1.2",
+        }})
+      else
+        should include("cs_method"=>"GET", "cs_host"=>"d111111abcdef8.cloudfront.net", "cs_uri_stem"=>"/index.html", "cs_protocol"=>"https", "cs_bytes"=>23)
+        should include("x_host_header"=>"d111111abcdef8.cloudfront.net")
+        should include("time_taken"=>0.001)
+        should include("x_edge_request_id"=>"SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==")
+        should include("agent"=>"Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36")
+
+        should include("x_forwarded_for"=>"text/html") # TODO the legacy pattern does not handle the long(er) format correctly
+      end
+    end
+
+  end
+
+end

data/spec/patterns/bacula_spec.rb

@@ -0,0 +1,367 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe_pattern "BACULA_LOG_MAX_CAPACITY", ['legacy', 'ecs-v1'] do
+
+  let(:message) do
+    'User defined maximum volume capacity 108,372,182,400 exceeded on device "FStorage" (/var/lib/bac/storage).'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      should include "bacula"=>{"volume"=>{"max_capacity"=>"108,372,182,400", "device"=>"FStorage", "path"=>"/var/lib/bac/storage"}}
+    else
+      should include("device"=>"FStorage")
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOG_END_VOLUME", ['legacy', 'ecs-v1'] do
+
+  let(:message) do
+    'End of medium on Volume "TestShortZN0014" Bytes=5,228,777 Blocks=82 at 21-Dec-2016 12:30.'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      should include "bacula"=>hash_including("volume"=>{"name"=>"TestShortZN0014", "bytes"=>"5,228,777", "blocks"=>"82"})
+      # bacula.timestamp is 'duplicate' information when the full BACULA_LOGLINE is matched
+      # we're keeping it as it includes year and might be slightly off the matched timestamp
+      should include "bacula"=>hash_including("timestamp"=>"21-Dec-2016 12:30")
+    else
+      should include("volume"=>"TestShortZN0014")
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_NEW_VOLUME
+
+  let(:message) do
+    '09-Jan 19:54 bacula-host JobId 265896: Created new Volume "FullAuto-8812" in catalog.'
+    # NOTE: we do not match full message log format that look like:
+    # 'Created new Volume="FullAuto-8812", Pool="FullFile", MediaType="FullFile" in catalog.'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '09-Jan 19:54'
+    if ecs_compatibility?
+      should include "bacula"=>{"volume"=>{"name"=>"FullAuto-8812"}, "job"=>{"id"=>"265896"}}
+      should include "host" => {"hostname"=>"bacula-host"}
+    else
+      should include("volume"=>"FullAuto-8812")
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_NEW_LABEL
+
+  let(:message) do
+    '25-Aug 10:50 bacula-sd JobId 24: Labeled new Volume "Vol-0018" on device "FileChgr1-Dev1" (/opt/bacula/disk).'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '25-Aug 10:50'
+    if ecs_compatibility?
+      should include "bacula"=>hash_including("volume"=>{"name"=>"Vol-0018", "device"=>"FileChgr1-Dev1", "path"=>"/opt/bacula/disk"})
+      should include "bacula"=>hash_including("job"=>{"id"=>"24"})
+      should include "host" => {"hostname"=>"bacula-sd"}
+    else
+      should include("volume"=>"Vol-0018", "device" => "FileChgr1-Dev1")
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_WROTE_LABEL
+
+  let(:message) do
+    '25-Aug 10:50 bacula-sd JobId 24: Wrote label to prelabeled Volume "Volume01" on device "Device01" (/dev/nst0)'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '25-Aug 10:50'
+    if ecs_compatibility?
+      should include "bacula"=>hash_including("volume"=>{"name"=>"Volume01", "device"=>"Device01", "path"=>"/dev/nst0"})
+    else
+      should include("jobid"=>"24")
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_NEW_MOUNT
+
+  let(:message) do
+    '24-Aug 01:54 crey-sd JobId 215534: New volume "DiffAuto-4861" mounted on device "vDrive-1" (/usr/local/bac/volumes) at 24-Aug-2015 01:54.'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '24-Aug 01:54'
+    if ecs_compatibility?
+      should include "bacula"=>hash_including("volume"=>{"name"=>"DiffAuto-4861", "device"=>"vDrive-1", "path"=>"/usr/local/bac/volumes"})
+    else
+      should include("device"=>"vDrive-1", "volume"=>"DiffAuto-4861", "hostname"=>"crey-sd", "jobid"=>"215534")
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_NOOPENDIR
+
+  let(:message) do
+    '24-Feb 16:36 starfury-fd JobId 3: Could not open directory "/root": ERR=Permission denied'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '24-Feb 16:36'
+    if ecs_compatibility?
+      should include "file"=>{"path"=>"/root"}
+      should include "error"=>{"message"=>"Permission denied"}
+    else
+      should include("berror"=>"Permission denied")
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_NOSTAT
+
+  let(:message) do
+    '15-Dec 17:50 u22.com JobId 13:      Could not stat /var/lib/bacula/bacula.sql: ERR=No such file or directory'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      should include "timestamp" => '15-Dec 17:50'
+      should include "file"=>{"path"=>"/var/lib/bacula/bacula.sql"}
+      should include "error"=>{"message"=>"No such file or directory"}
+    else
+      # NOTE: not matching due BACULA_HOST
+      # should include "bts" => '15-Dec 17:50'
+      # should include "berror"=>"No such file or directory"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_ALL_RECORDS_PRUNED
+
+  let(:message) do
+    '12-Apr 14:23 VU0EM005: All records pruned from Volume "06D125L3"; marking it "Purged"'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '12-Apr 14:23'
+    if ecs_compatibility?
+      should include "bacula"=>{"volume"=>{"name"=>"06D125L3"}},
+                     "host"=>{"hostname"=>"VU0EM005"}
+    else
+      should include "hostname"=>"VU0EM005", "volume"=>"06D125L3"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_PRUNED_JOBS
+
+  let(:message) do
+    '29-Jan 04:16 lbu02-dir: Pruned 24 Jobs for client uni-horn from catalog.'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '29-Jan 04:16'
+    if ecs_compatibility?
+      should include "bacula"=>{"client"=>{"name"=>"uni-horn"}}, "host"=>{"hostname"=>"lbu02-dir"}
+    else
+      should include "hostname"=>"lbu02-dir", "client"=>"uni-horn"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_STARTJOB
+
+  let(:message) do
+    '06-Mar 20:00 srvbkp-dir JobId 1075: Start Backup JobId 1075, Job=srv1-bind.2018-03-06_20.00.01_05'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '06-Mar 20:00'
+    if ecs_compatibility?
+      should include "bacula"=>{"job"=>{"name"=>"srv1-bind.2018-03-06_20.00.01_05", "id"=>"1075"}}
+    else
+      should include "job"=>"srv1-bind.2018-03-06_20.00.01_05", "jobid"=>"1075"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_DIFF_FS
+
+  let(:message) do
+    '01-Feb 00:34 ohms-fd JobId 1662:      /var/spool/bareos is a different filesystem. Will not descend from /var into it.'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '01-Feb 00:34'
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_JOBEND
+
+  let(:message) do
+    '28-Aug 21:55 bacula-sd JobId 16: Job write elapsed time = 00:00:01, Transfer rate = 0  Bytes/second'
+  end
+
+  it 'matches' do
+    should include (ecs_compatibility? ? "timestamp" : "bts") => '28-Aug 21:55'
+    if ecs_compatibility?
+      should include "bacula"=>{"job"=>{"elapsed_time"=>"00:00:01", "id"=>"16"}}
+    else
+      should include "jobid"=>"16", "elapsed" => "00:00:01"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_VOLUME_PREVWRITTEN
+
+  let(:message) do
+    '17-Jan-2003 16:45 home-sd: Volume test01 previously written, moving to end of data.'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      should include "timestamp" => '17-Jan-2003 16:45'
+      should include "bacula"=>{"volume"=>{"name"=>"test01"}}
+    else
+      # fails to match (due timestamp format)
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOG_READYAPPEND", ['legacy', 'ecs-v1'] do
+
+  let(:message) do
+    'Ready to append to end of Volume "F-0032" size=97835302'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      should include "bacula"=>{"volume"=>{"name"=>"F-0032", "size"=>97835302}}
+    else
+      should include "volume"=>"F-0032"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_CLIENT_RBJ
+
+  let(:message) do
+    '01-Aug 13:30 toe-fd JobId 686: shell command: run ClientRunBeforeJob "/etc/bacula/cbe_hanfs.sh /mnt/baxter/fs1"'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      should include "bacula"=>{"job"=>{"id"=>"686", "client_run_before_command"=>'/etc/bacula/cbe_hanfs.sh /mnt/baxter/fs1'}}
+    else
+      should include "jobid"=>"686", "runjob"=>"/etc/bacula/cbe_hanfs.sh /mnt/baxter/fs1"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_FATAL_CONN
+
+  let(:message) do
+    '11-Nov 13:28 bacula-dir JobId 11: Fatal error: bsock.c:133 Unable to connect to Client: dc0-fd on dc0.teamworld.com:9102. ERR=Connection refused'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      should include "client"=>{"address"=>"dc0.teamworld.com", "port"=>9102},
+                     "bacula"=>hash_including("client"=>{"name"=>"dc0-fd"}),
+                      "error"=>{"message"=>"Connection refused"}
+    else
+      should include "client"=>"dc0-fd", "berror"=>"Connection refused"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_NO_AUTH
+
+  let(:message) do
+    '16-May 11:59 samy-dir JobId 0: Fatal error: Unable to authenticate with File daemon at "cardam.home.domain:9102". Possible causes:'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      # NOTE: due a grok bug port:int type-casting does not work :
+      #should include "client"=>{"address"=>"cardam.home.domain", "port"=>9102}
+      expect( subject['client'] ).to be_a Hash
+      expect( subject['client']['address'] ).to eql 'cardam.home.domain'
+      expect( subject['client']['port'].to_i ).to eql 9102
+    else
+      # does not match due client address:port
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_CANCELLING
+
+  let(:message) do
+    '03-Aug 06:20 DIRECTOR JobId 316677: Cancelling duplicate JobId=316646.'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      expect( subject ).to include "bacula" => hash_including("job" => {'id' => '316677', 'other_id' => '316646'})
+    else
+      expect( subject ).to include "jobid" => "316677"
+    end
+  end
+
+end
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_MARKCANCEL
+
+  let(:message) do
+    '09-Aug 15:14 InternetServer-sd JobId 122971, Job nyi_maildir.2013-03-03_22.00.00_51 marked to be canceled.'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      expect( subject ).to include "bacula" => hash_including(
+          "job" => {'id' => '122971', 'name' => 'nyi_maildir.2013-03-03_22.00.00_51'})
+    else
+      expect( subject ).to include "job" => "nyi_maildir.2013-03-03_22.00.00_51"
+    end
+  end
+
+end
+
+
+describe_pattern "BACULA_LOGLINE", ['legacy', 'ecs-v1'] do # BACULA_LOG_FATAL_CONN
+
+  let(:message) do
+    '25-Aug 09:02 marlin2-dir JobId 10783: Fatal Error: JobId 10782 already running. Duplicate job not allowed.'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      expect( subject ).to include "bacula" => hash_including("job" => {'id' => '10783', 'other_id' => '10782'})
+    else
+      # NOTE: not matching due expecting 'error' instead of 'Error' in "Fatal Error: JobId ..."
+    end
+  end
+
+end