logstash-patterns-core 4.1.0 → 4.3.1

data/spec/patterns/haproxy_spec.rb

@@ -2,19 +2,60 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
-describe "HAPROXY" do
+describe_pattern "HAPROXYHTTP", ['legacy', 'ecs-v1'] do
 
-  let(:haproxyhttp_pattern)  { "HAPROXYHTTP" }
+  context "log line from raw syslog line" do
 
-  context "Parsing HAPROXY log line from raw syslog line" do
+    let(:message) do
+      'Dec  9 13:01:26 localhost haproxy[28029]: 127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"'
+    end
+
+    it "matches" do
+      if ecs_compatibility?
+        expect(subject).to include("timestamp"=>"Dec  9 13:01:26")
+        expect(subject).to include("host"=>{"hostname"=>"localhost"})
+        expect(subject).to include("process"=>{"pid"=>28029, "name"=>"haproxy"})
+        expect(subject).to include("source"=>{"port"=>39759, "address"=>"127.0.0.1", "bytes"=>83285})
+        expect(subject).to include("haproxy" => hash_including("request_date"=>"09/Dec/2013:12:59:46.633"))
+        expect(subject).to include("haproxy" => hash_including("frontend_name"=>"loadbalancer", "backend_name"=>"default", "server_name"=>"instance8"))
+        expect(subject).to include("haproxy" => hash_including(
+            "total_waiting_time_ms"=>51536, "connection_wait_time_ms"=>1, "total_time_ms"=>"99627",
+            "http" => hash_including("request"=>hash_including("time_wait_ms"=>0, "time_wait_without_data_ms"=>48082))
+        ))
+        expect(subject).to include("http" => hash_including("response"=>{"status_code"=>200}))
+
+        expect(subject).to include("haproxy" => hash_including("termination_state"=>"----"))
+
+        expect(subject).to include("haproxy" => hash_including("connections"=>{"active"=>87, "frontend"=>87, "backend"=>87, "server"=>1, "retries"=>0}))
+        expect(subject).to include("haproxy" => hash_including("backend_queue"=>67, "server_queue"=>0))
 
-    let(:value) { 'Dec  9 13:01:26 localhost haproxy[28029]: 127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"' }
-    subject     { grok_match(haproxyhttp_pattern, value) }
+        expect(subject).to include("http" => hash_including("request" => {"method"=>'GET'}, "version" => '1.1'))
+
+        expect(subject).to include("url" => { "original"=>"/path/to/image", "path"=>"/path/to/image" })
+      else
+        expect(subject).to include("syslog_timestamp" => "Dec  9 13:01:26")
+        expect(subject).to include("syslog_server" => "localhost")
+        expect(subject).to include("http_request" => "/path/to/image", "http_status_code" => "200", "http_verb" => "GET", "http_version" => "1.1")
+        expect(subject).to include("program" => "haproxy")
+        expect(subject).to include("client_ip" => "127.0.0.1")
+        expect(subject).to include("http_verb" => "GET")
+        expect(subject).to include("server_name" => "instance8")
+      end
+    end
+
+    it "has no captured cookies" do
+      if ecs_compatibility?
+        expect((subject['haproxy']['http']['request'] || {}).keys).to_not include('captured_cookie')
+        expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_cookie')
+      end
+    end
 
-    it { should include("program" => "haproxy") }
-    it { should include("client_ip" => "127.0.0.1") }
-    it { should include("http_verb" => "GET") }
-    it { should include("server_name" => "instance8") }
+    it "includes header captures" do
+      if ecs_compatibility?
+        expect((subject['haproxy']['http'])).to include('request' => hash_including('captured_headers' => '77.24.148.74'))
+        expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_headers')
+      end
+    end
 
     it "generates a message field" do
       expect(subject["message"]).to include("loadbalancer default/instance8")
@@ -22,38 +63,222 @@ describe "HAPROXY" do
 
   end
 
-  context "Parsing HAPROXY log line from raw syslog line with ISO8601 timestamp" do
+  context "log line (without headers) from raw syslog line with ISO8601 timestamp" do
 
-    let(:value) { '2015-08-26T02:09:48+02:00 localhost haproxy[28029]: 127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"' }
-    subject     { grok_match(haproxyhttp_pattern, value) }
+    let(:message) do
+      '2015-08-26T02:09:48+02:00 localhost haproxy[14389]: 5.196.2.38:39527 [03/Nov/2015:06:25:25.105] services~ def/api 4599/0/0/428/5027 304 320 - - ---- 1/1/0/1/0 0/0 "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1"'
+    end
 
-    it { should include("program" => "haproxy") }
-    it { should include("client_ip" => "127.0.0.1") }
-    it { should include("http_verb" => "GET") }
-    it { should include("server_name" => "instance8") }
+    it "matches" do
+      if ecs_compatibility?
+        expect(subject).to include("timestamp"=>"2015-08-26T02:09:48+02:00")
+        expect(subject).to include("host"=>{"hostname"=>"localhost"})
+        expect(subject).to include("process"=>{"pid"=>14389, "name"=>"haproxy"})
 
-    it "generates a message field" do
-      expect(subject["message"]).to include("loadbalancer default/instance8")
+        expect(subject).to include("haproxy" => hash_including("connections"=>{"active"=>1, "frontend"=>1, "backend"=>0, "server"=>1, "retries"=>0}))
+        expect(subject).to include("haproxy" => hash_including("backend_queue"=>0, "server_queue"=>0))
+
+        expect(subject).to include("haproxy" => hash_including("frontend_name"=>"services~"))
+
+        expect(subject).to include("http"=>{"response"=>{"status_code"=>304}, "version"=>"1.1", "request"=>{"method"=>"GET"}})
+        expect(subject).to include("url"=>hash_including("path"=>"/component---src-pages-index-js-4b15624544f97cf0bb8f.js"))
+      else
+        expect(subject).to include("program" => "haproxy")
+        expect(subject).to include("client_ip" => "5.196.2.38")
+        expect(subject).to include("http_verb" => "GET")
+        expect(subject).to include("server_name" => "api")
+      end
+    end
+
+    it "has no header captures" do
+      if ecs_compatibility?
+        expect((subject['haproxy']['http']['request'] || {}).keys).to_not include('captured_headers')
+        expect((subject['haproxy']['http']['response'] || {}).keys).to_not include('captured_headers')
+      end
     end
 
   end
 
-  let(:haproxyhttpbase_pattern)  { "HAPROXYHTTPBASE" }
+  context 'log line with both request/response headers' do
 
-  context "Parsing HAPROXY log line without syslog specific enteries.  This mimics an event coming from a syslog input." do
+    let(:message) do
+      'Jul 30 09:03:52 home.host haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} "GET http://192.168.0.12:8080/serv/login.php?lang=en&profile=2 HTTP/1.1"'
+    end
 
-    let(:value) { '127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET /path/to/image HTTP/1.1"' }
-    subject     { grok_match(haproxyhttpbase_pattern, value) }
+    it "matches" do
+      if ecs_compatibility?
+        expect(subject).to include("timestamp"=>"Jul 30 09:03:52")
+        expect(subject).to include("host"=>{"hostname"=>"home.host"})
 
-    # Assume 'program' would be matched by the syslog input.
-    it { should include("client_ip" => "127.0.0.1") }
-    it { should include("http_verb" => "GET") }
-    it { should include("server_name" => "instance8") }
+        expect(subject).to include("haproxy" => hash_including("frontend_name"=>"incoming~"))
 
-    it "generates a message field" do
-      expect(subject["message"]).to include("loadbalancer default/instance8")
+        expect(subject).to include("http"=>{"response"=>{"status_code"=>304}, "version"=>"1.1", "request"=>{"method"=>"GET"}})
+        expect(subject).to include("url"=>hash_including("scheme"=>"http", "domain"=>"192.168.0.12", "port"=>8080,
+                                             "path"=>"/serv/login.php", "query"=>"lang=en&profile=2",
+                                             "original"=>"http://192.168.0.12:8080/serv/login.php?lang=en&profile=2"))
+      else
+        expect(subject).to include("client_ip" => "1.2.3.4")
+        expect(subject).to include("http_verb" => "GET")
+      end
+    end
+
+    it "has header captures" do
+      if ecs_compatibility?
+        expect((subject['haproxy']['http']['request'])).to include('captured_headers' => 'docs.example.internal||')
+        expect((subject['haproxy']['http']['response'])).to include('captured_headers' => '|||')
+      end
+    end
+
+  end
+
+  context 'BADREQ/NOSRV log line' do
+
+    let(:message) do
+      'Jul 18 17:05:30 localhost haproxy[8247]: 188.223.50.7:51940 [18/Jul/2011:17:05:24.339] http_proxy_ads http_proxy_ads/<NOSRV> -1/-1/-1/-1/6001 408 212 - - cR-- 100/89/0/0/0 0/0 "<BADREQ>"'
+    end
+
+    it "matches" do
+      if ecs_compatibility?
+        expect(subject).to include("timestamp"=>"Jul 18 17:05:30")
+
+        expect(subject).to include("haproxy" => hash_including("frontend_name"=>"http_proxy_ads"))
+        expect(subject).to include("haproxy" => hash_including("backend_name"=>"http_proxy_ads"))
+        expect(subject['haproxy'].keys).to_not include('server_name')
+        expect(subject).to include("http"=>{"response"=>{"status_code"=>408}})
+        expect(subject['haproxy'].keys).to_not include("total_waiting_time_ms", "connection_wait_time_ms")
+        expect(subject).to include("haproxy" => hash_including("total_time_ms"=>"6001"))
+        expect(subject).to include("source" => hash_including("bytes"=>212))
+        expect(subject).to include("haproxy" => hash_including("termination_state"=>"cR--"))
+        expect(subject.keys).to_not include("url")
+      else
+        expect(subject).to include("backend_name"=>"http_proxy_ads", "frontend_name"=>"http_proxy_ads", "server_name"=>"<NOSRV>")
+        expect(subject).to include("http_status_code"=>"408")
+        expect(subject).to include("time_backend_connect"=>"-1", "time_queue"=>"-1", "time_backend_response"=>"-1")
+        expect(subject).to include("captured_request_cookie"=>"-", "captured_response_cookie"=>"-")
+        expect(subject).to include("bytes_read"=>"212")
+        expect(subject).to include("termination_state"=>"cR--")
+      end
+    end
+
+  end
+
+end
+
+describe_pattern "HAPROXYHTTPBASE", ['ecs-v1', 'legacy'] do
+
+  context "log line without syslog specific entries" do # This mimics an event coming from a syslog input.
+
+    let(:message) do
+      '127.0.0.1:39759 [09/Dec/2013:12:59:46.633] loadbalancer default/instance8 0/51536/1/48082/+99627 200 83285 - - ---- 87/87/87/1/0 0/67 {77.24.148.74} "GET / HTTP/1.1"'
+    end
+
+    it 'matches' do
+      if ecs_compatibility?
+        expect(subject).to include("source"=>{"port"=>39759, "address"=>"127.0.0.1", "bytes"=>83285})
+        expect(subject).to include("haproxy"=>hash_including("server_queue"=>0,
+                                  "http"=>{
+                                      "request"=>{"time_wait_ms"=>0, "captured_headers"=>"77.24.148.74", "time_wait_without_data_ms"=>48082}
+                                  },
+
+                                  # NOTE: this is why we do not type-cast to :int
+                                  # a '+' sign is prepended before the value, indicating that the final one will be larger
+                                  "total_time_ms" => "+99627"
+        ))
+        expect(subject).to include("url"=>{"path"=>"/", "original"=>"/"})
+      else
+        # Assume 'program' would be matched by the syslog input.
+        expect(subject).to include("client_ip" => "127.0.0.1")
+        expect(subject).to include("server_name" => "instance8")
+        expect(subject).to include("http_verb" => "GET", "http_request"=>"/", "http_version" => '1.1')
+        expect(subject).to include("time_duration" => "+99627")
+      end
+    end
+
+  end
+
+  context "(incomplete) log line that is truncated and thus not ending with a double quote or HTTP version" do
+
+    let(:message) do
+      'Jul 31 22:20:22 loadbalancer haproxy[1190]: 203.0.113.54:59968 [31/Jul/2017:22:20:22.447] loadbalancer default/instance8 135/0/1/19/156 200 1015 - - --VR 8/8/0/0/0 0/0 "GET /path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss'
+    end
+
+    it 'matches' do
+      if ecs_compatibility?
+        # due compatibility with the legacy pattern we match the incomplete "REQUEST LINE ... (wout the ending '"')
+        expect(subject).to include("http"=>{"response"=>{"status_code"=>200}, "request"=>{"method"=>"GET"}})
+        expect(subject).to include("url"=>hash_including("original"=>"/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss"))
+      else
+        expect(subject).to include("client_ip" => "203.0.113.54")
+        expect(subject).to include("http_verb" => "GET")
+        expect(subject).to include("server_name" => "instance8")
+        expect(subject).to include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss")
+        expect(subject).to_not have_key("http_version")
+      end
+    end
+
+  end
+
+
+  context "connect line with host:port url" do
+
+    let(:message) do
+      'Nov  4 08:32:18 debian10 haproxy[3666]: 127.0.0.1:34500 [04/Nov/2020:08:32:18.194] samplefrontend backendnodes/node1 0/0/0/0/0 405 501 - - ---- 1/1/0/1/0 0/0 "CONNECT localhost:8080 HTTP/1.1"'
+    end
+
+    it 'matches' do
+      if ecs_compatibility?
+        expect(subject).to include("http"=>hash_including("request"=>{"method"=>"CONNECT"}))
+        expect(subject).to include("url"=>{"port"=>8080, "original"=>"localhost:8080", "domain"=>"localhost"})
+      else
+        expect(subject).to include("http_verb" => "CONNECT")
+        expect(subject).to include("http_host" => "localhost:8080")
+      end
     end
 
   end
 
 end
+
+describe_pattern "HAPROXYTCP", ['legacy', 'ecs-v1'] do
+
+  let(:message) do
+    'Sep 20 15:44:23 127.0.0.1 haproxy[25457]: 127.0.0.1:40962 [20/Sep/2018:15:44:23.285] main app/<NOSRV> -1/-1/1 212 SC 1/1/0/0/0 0/0'
+  end
+
+  it 'matches' do
+    if ecs_compatibility?
+      expect(subject).to include(
+                            "timestamp"=>"Sep 20 15:44:23",
+                            "host"=>{"hostname"=>"127.0.0.1"},
+                            "process"=>{"pid"=>25457, "name"=>"haproxy"},
+                            "source"=>{"port"=>40962, "address"=>"127.0.0.1", "bytes"=>212},
+                            "haproxy"=>{
+                                "request_date"=>"20/Sep/2018:15:44:23.285",
+                                "frontend_name"=>"main", "backend_name"=>"app",
+                                "total_time_ms"=>"1",
+                                "termination_state"=>"SC",
+                                "connections"=>{"active"=>1, "backend"=>0, "retries"=>0, "server"=>0, "frontend"=>1},
+                                "server_queue"=>0, "backend_queue"=>0
+                            })
+    else
+      expect(subject).to include(
+                            "syslog_timestamp"=>"Sep 20 15:44:23",
+                            "syslog_server"=>"127.0.0.1",
+                            "program"=>"haproxy", "pid"=>"25457",
+                            "client_ip"=>"127.0.0.1", "client_port"=>"40962",
+                            "accept_date"=>"20/Sep/2018:15:44:23.285",
+                            "frontend_name"=>"main",
+                            "backend_name"=>"app",
+                            "server_name"=>"<NOSRV>",
+                            "time_backend_connect"=>"-1",
+                            "time_queue"=>"-1",
+                            "time_duration"=>"1",
+                            "bytes_read"=>"212",
+                            "termination_state"=>"SC",
+                            "actconn"=>"1", "feconn"=>"1", "beconn"=>"0", "backend_queue"=>"0", "retries"=>"0",
+                            "srv_queue"=>"0", "srvconn"=>"0",
+                            )
+    end
+  end
+
+end

data/spec/patterns/httpd_spec.rb

@@ -2,126 +2,304 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
-describe "HTTPD_COMBINEDLOG" do
+describe_pattern "HTTPD_COMBINEDLOG", ['legacy', 'ecs-v1'] do
 
-  context "HTTPD_COMBINEDLOG", "Typical test case" do
+  context "typical test case" do
 
-    let(:value) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'}
+    let(:message) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'}
 
-    it "generates the clientip field" do
-      expect(grok_match(subject, value)).to include(
-        'clientip' => '83.149.9.216',
-        'verb' => 'GET',
-        'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
-        'httpversion' => '1.1',
-        'response' => '200',
-        'bytes' => '203023',
-        'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
-        'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
-      )
+    it "matches" do
+      if ecs_compatibility?
+        expect(grok).to include(
+                            "http" => {
+                                "request" => {
+                                    "method" => "GET",
+                                    "referrer" => "http://semicomplete.com/presentations/logstash-monitorama-2013/"
+                                },
+                                "response" => {
+                                    "body" => { "bytes" => 203023 },
+                                    "status_code" => 200
+                                },
+                                "version"=>"1.1"
+                            },
+                            "source" => { "address" => "83.149.9.216" },
+                            "url" => { "original" => "/presentations/logstash-monitorama-2013/images/kibana-search.png" },
+                            "user_agent" => { "original" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" }
+                        )
+      else
+        expect(grok).to include(
+                            'clientip' => '83.149.9.216',
+                            'verb' => 'GET',
+                            'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
+                            'httpversion' => '1.1',
+                            'response' => '200',
+                            'bytes' => '203023',
+                            'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
+                            'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
+                        )
+      end
+    end
+
+    it "does not capture 'null' fields" do
+      if ecs_compatibility?
+        expect(grok.keys).to_not include('user') # 'user' => 'name'
+        expect(grok.keys).to_not include('apache') # apache.access.user.identity
+      else
+        expect(grok).to include('auth' => '-', 'ident' => '-')
+      end
+    end
+
+  end
+
+  context "email address in auth field" do
+
+    let(:message) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'}
+
+    it "gets captured" do
+      if ecs_compatibility?
+        expect(grok).to include("user" => { 'name' => "username@example.com" })
+      else
+        expect(grok).to include("auth" => "username@example.com")
+      end
     end
 
   end
 
-  context "HTTPD_COMBINEDLOG", "Email address in auth field" do
+  context 'sample OPTIONS line' do
 
-    let(:value) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'}
+    let(:message) { '83.149.9.216 - a.user [11/Jan/2020:23:05:27 +0100] "OPTIONS /remote.php/ HTTP/1.1" - 7908 "-" "monitoring-client (v2.2)"' }
 
-    it "generates the clientip field" do
-      expect(grok_match(subject, value)).to include("auth" => "username@example.com")
+    it 'matches' do
+      if ecs_compatibility?
+        expect(grok).to include("http" => hash_including("response" => hash_including("body" => { "bytes" => 7908 })))
+        expect(grok).to include("http" => hash_including("request" => { "method" => "OPTIONS" }, "version" => "1.1"))
+        expect(grok).to include(
+                            "url" => { "original" => "/remote.php/" },
+                            "user_agent" => { "original" => "monitoring-client (v2.2)" }
+                        )
+      else
+        expect(grok).to include("verb" => "OPTIONS", 'request' => '/remote.php/', 'httpversion' => '1.1', "bytes" => '7908')
+      end
+    end
+
+    it 'does not capture optional response code' do
+      if ecs_compatibility?
+        expect(grok['http']['response'].keys).to_not include("status_code")
+      else
+        expect(grok.keys).to_not include("response")
+      end
     end
 
   end
 
 end
 
-describe "HTTPD_ERRORLOG" do
+describe_pattern "HTTPD_ERRORLOG", ['legacy', 'ecs-v1'] do
 
-  context "HTTPD_ERRORLOG", "matches a full httpd 2.4 message" do
-    let(:value) {
+  context "a full httpd 2.4 message" do
+    let(:message) do
       "[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product"
-    }
-    it "generates the fields" do
+    end
 
-      expect(grok_match(subject, value)).to include(
-        'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
-        'module' => 'proxy_fcgi',
-        'loglevel' => 'error',
-        'pid' => '28787',
-        'tid' => '140169587934976',
-        'proxy_errorcode' => '70008',
-        'proxy_message' => 'Partial results are valid but processing is incomplete',
-        'clientip' => '58.13.45.166',
-        'clientport' => '59307',
-        'errorcode' => 'AH01075',
-        'message' => [ value, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ],
-      )
+    it "generates the fields" do
+      expect(grok).to include('timestamp' => 'Mon Aug 31 09:30:48.958285 2015')
+      if ecs_compatibility?
+        expect(grok).to include("log" => { "level" => "error" })
+        expect(grok).to include("process" => { "pid" => 28787, "thread" => { "id" => 140169587934976 } })
+        expect(grok).to include("source" => { "address"=>"58.13.45.166", "port" => 59307 })
+        expect(grok).to include("error" => { "code" => 'AH01075' })
+        expect(grok).to include("apache" => { "error" => {
+            "module" => "proxy_fcgi",
+            "proxy" => { "error" => { "code" => '70008', "message" => "Partial results are valid but processing is incomplete" }}}
+        })
+      else
+        expect(grok).to include(
+                            'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
+                            'module' => 'proxy_fcgi',
+                            'loglevel' => 'error',
+                            'pid' => '28787',
+                            'tid' => '140169587934976',
+                            'proxy_errorcode' => '70008',
+                            'proxy_message' => 'Partial results are valid but processing is incomplete',
+                            'clientip' => '58.13.45.166',
+                            'clientport' => '59307',
+                            'errorcode' => 'AH01075'
+                            )
+      end
+      expect(grok).to include('message' => [ message, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ])
     end
   end
 
-  context "HTTPD_ERRORLOG", "matches a httpd 2.2 log message" do
-    let(:value) {
+  context "a httpd 2.2 log message" do
+    let(:message) do
       "[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com"
-    }
+    end
+
     it "generates the fields" do
-      expect(grok_match(subject, value)).to include(
-        'timestamp' => 'Mon Aug 31 16:27:04 2015',
-        'loglevel' => 'error',
-        'clientip' => '10.17.42.3',
-        'message' => [ value, 'Premature end of script headers: example.com' ]
-      )
+      if ecs_compatibility?
+        expect(grok).to include(
+                            "timestamp"=>"Mon Aug 31 16:27:04 2015",
+                            "log"=>{"level"=>"error"},
+                            "source"=>{"address"=>"10.17.42.3"})
+        expect(grok.keys).to_not include("error") # error.code
+      else
+        expect(grok).to include(
+                            'timestamp' => 'Mon Aug 31 16:27:04 2015',
+                            'loglevel' => 'error',
+                            'clientip' => '10.17.42.3'
+                        )
+        expect(grok.keys).to_not include('errorcode')
+      end
+      expect(grok).to include('message' => [ message, 'Premature end of script headers: example.com' ])
     end
   end
 
-  context "HTTPD_ERRORLOG", "matches a short httpd 2.4 message" do
-    let(:value) {
+  context "a short httpd 2.4 message" do
+    let(:value1) {
       "[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'"
     }
     it "generates the fields" do
-      expect(grok_match(subject, value)).to include(
-        'timestamp' => 'Mon Aug 31 07:15:38.664897 2015',
-        'module' => 'proxy_fcgi',
-        'loglevel' => 'error',
-        'pid' => '28786',
-        'tid' => '140169629898496',
-        'clientip' => '81.139.1.34',
-        'clientport' => '52042',
-        'errorcode' => 'AH01071',
-        'message' => [ value, "Got error 'Primary script unknown\n'" ]
-      )
+      match_result = grok_match(pattern, value1)
+      expect(match_result).to include('timestamp' => 'Mon Aug 31 07:15:38.664897 2015')
+      if ecs_compatibility?
+        expect(match_result).to include(
+                                    "apache"=>{"error"=>{"module"=>"proxy_fcgi"}},
+                                    "log"=>{"level"=>"error"},
+                                    "process"=>{"pid"=>28786, "thread"=>{"id"=>140169629898496}},
+                                    "source"=>{"address"=>"81.139.1.34", "port"=>52042},
+                                    "error"=>{"code"=>"AH01071"},
+                                )
+      else
+        expect(match_result).to include(
+                                    'module' => 'proxy_fcgi',
+                                    'loglevel' => 'error',
+                                    'pid' => '28786',
+                                    'tid' => '140169629898496',
+                                    'clientip' => '81.139.1.34',
+                                    'clientport' => '52042',
+                                    'errorcode' => 'AH01071'
+                                )
+      end
+      expect(match_result).to include('message' => [ value1, "Got error 'Primary script unknown\n'" ])
+    end
+
+    let(:value2) {
+      "[Thu Apr 27 10:39:46.719636 2017] [php7:notice] [pid 17] [client 10.255.0.3:49580] Test error log record"
+    }
+	  it "generates the fields" do
+      match_result = grok_match(pattern, value2)
+      expect(match_result).to include('timestamp' => 'Thu Apr 27 10:39:46.719636 2017')
+      if ecs_compatibility?
+        expect(match_result).to include(
+                                    "apache"=>{"error"=>{"module"=>"php7"}},
+                                    "log"=>{"level"=>"notice"},
+                                    "process"=>{"pid"=>17},
+                                    "source"=>{"port"=>49580, "address"=>"10.255.0.3"}
+                                )
+      else
+        expect(match_result).to include(
+                                    'module' => 'php7',
+                                    'loglevel' => 'notice',
+                                    'pid' => '17',
+                                    'clientip' => '10.255.0.3',
+                                    'clientport' => '49580'
+                                )
+      end
+      expect(match_result).to include('message' => [ value2, "Test error log record" ])
     end
   end
 
-  context "HTTPD_ERRORLOG", "matches an httpd 2.4 restart" do
+  context "a httpd 2.4 restart message" do
     let(:value1) {
       "[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations"
     }
     it "generates the fields" do
-      expect(grok_match(subject, value1)).to include(
-        'timestamp' => 'Mon Aug 31 06:29:47.406518 2015',
-        'module' => 'mpm_event',
-        'loglevel' => 'notice',
-        'pid' => '24968',
-        'tid' => '140169861986176',
-        'errorcode' => 'AH00489',
-        'message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ]
-      )
+      match_result = grok_match(pattern, value1)
+      expect(match_result).to include('timestamp' => 'Mon Aug 31 06:29:47.406518 2015')
+      if ecs_compatibility?
+        expect(match_result).to include(
+                                    "apache"=>{"error"=>{"module"=>"mpm_event"}},
+                                    "log"=>{"level"=>"notice"},
+                                    "process"=>{"pid"=>24968, "thread"=>{"id"=>140169861986176}},
+                                    "error"=>{"code"=>"AH00489"}
+                                )
+
+      else
+        expect(match_result).to include(
+                                    'module' => 'mpm_event',
+                                    'loglevel' => 'notice',
+                                    'pid' => '24968',
+                                    'tid' => '140169861986176',
+                                    'errorcode' => 'AH00489'
+                                )
+      end
+      expect(match_result).to include('message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ])
     end
 
     let(:value2) {
       "[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'"
     }
     it "generates the fields" do
-      expect(grok_match(subject, value2)).to include(
-        'timestamp' => 'Mon Aug 31 06:29:47.406530 2015',
-        'module' => 'core',
-        'loglevel' => 'notice',
-        'pid' => '24968',
-        'tid' => '140169861986176',
-        'errorcode' => 'AH00094',
-        'message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ]
-      )
+      match_result = grok_match(pattern, value2)
+      expect(match_result).to include('timestamp' => 'Mon Aug 31 06:29:47.406530 2015')
+      if ecs_compatibility?
+        expect(match_result).to include(
+                                    "apache"=>{"error"=>{"module"=>"core"}},
+                                    "log"=>{"level"=>"notice"},
+                                    "process"=>{"pid"=>24968, "thread"=>{"id"=>140169861986176}},
+                                    "error"=>{"code"=>"AH00094"}
+                                )
+      else
+        expect(match_result).to include(
+                                    'module' => 'core',
+                                    'loglevel' => 'notice',
+                                    'pid' => '24968',
+                                    'tid' => '140169861986176',
+                                    'errorcode' => 'AH00094'
+                                )
+      end
+      expect(match_result).to include('message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ])
+    end
+  end
+
+  context "a httpd 2.4 message witout module" do
+    let(:message) do
+      "[Tue Apr 14 14:27:52.605084 2020] [:error] [pid 5688] [client 192.168.10.110:8196] script '/login/wp-login.php' not found or unable to stat"
+    end
+
+    it "matches" do
+      expect(grok).to include('timestamp' => 'Tue Apr 14 14:27:52.605084 2020')
+      if ecs_compatibility?
+        expect(grok).to include("log"=>{"level" => "error"})
+        expect(grok).to include("process"=>{"pid" => 5688})
+        expect( ((grok['apache'] || {})['error'] || {}).keys ).to_not include('module')
+      else
+        expect(grok).to include('loglevel' => 'error', 'pid' => '5688')
+      end
     end
   end
 
+  context 'a debug message' do
+    let(:message) do
+      '[Fri Feb 01 22:03:08.319124 2019] [authz_core:debug] [pid 9:tid 140597881775872] mod_authz_core.c(820): [client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted'
+    end
+
+    it 'matches imperfectly (legacy)' do
+      if ecs_compatibility?
+        pending
+        raise NotImplementedError, "TODO: would be nice to 'improve' matching on these debug logs as well"
+      else
+        expect(grok).to include({
+                                    "timestamp"=>"Fri Feb 01 22:03:08.319124 2019",
+                                    "module"=>"authz_core",
+                                    "loglevel"=>"debug",
+                                    "pid"=>"9",
+                                    "tid"=>"140597881775872",
+                                    "errorcode"=>"mod_authz_core.c(820)",
+                                    "message"=>[message, "[client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted"]
+                                })
+      end
+    end
+  end
+  
 end