logstash-patterns-core 4.2.0 → 4.3.0

data/spec/patterns/shorewall_spec.rb

@@ -2,89 +2,100 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
-describe "SHOREWALL" do
-
-  let(:pattern)    { "SHOREWALL" }
+describe_pattern "SHOREWALL", ['legacy', 'ecs-v1'] do
 
   context "parsing a message with OUT interface" do
 
-    let(:value) { "May 28 17:23:25 myHost kernel: [3124658.791874] Shorewall:FORWARD:REJECT:IN=eth2 OUT=eth2 SRC=1.2.3.4 DST=1.2.3.4 LEN=141 TOS=0x00 PREC=0x00 TTL=63 ID=55251 PROTO=UDP SPT=5353 DPT=5353 LEN=121" }
-
-    subject     { grok_match(pattern, value) }
-
-    it { should include("timestamp" => "May 28 17:23:25") }
-
-    it { should include("nf_host" => "myHost") }
-
-    it { should include("nf_action1" => "FORWARD") }
-
-    it { should include("nf_action2" => "REJECT") }
-
-    it { should include("nf_in_interface" => "eth2") }
-
-    it { should include("nf_out_interface" => "eth2") }
-
-    it { should include("nf_src_ip" => "1.2.3.4") }
-
-    it { should include("nf_dst_ip" => "1.2.3.4") }
-
-    it { should include("nf_len" => "141") }
-
-    it { should include("nf_tos" => "0x00") }
-
-    it { should include("nf_prec" => "0x00") }
-
-    it { should include("nf_ttl" => "63") }
-
-    it { should include("nf_id" => "55251") }
-
-    it { should include("nf_protocol" => "UDP") }
-
-    it { should include("nf_src_port" => "5353") }
+    let(:message) do
+      "May 28 17:23:25 myHost kernel: [3124658.791874] Shorewall:FORWARD:REJECT:" +
+      "IN=eth2 OUT=eth2 SRC=1.2.3.4 DST=192.168.0.10 LEN=141 TOS=0x00 PREC=0x00 TTL=63 ID=55251 PROTO=UDP SPT=5353 DPT=5335 LEN=121"
+    end
+
+    it 'matches' do
+      expect(subject).to include("timestamp" => "May 28 17:23:25")
+      if ecs_compatibility?
+        expect(subject).to include(
+                            "observer"=>{"hostname"=>"myHost", "ingress"=>{"interface"=>{"name"=>"eth2"}}, "egress"=>{"interface"=>{"name"=>"eth2"}}},
+                            "shorewall"=>{'firewall'=>{"type"=>"FORWARD", "action"=>"REJECT"}},
+                            "iptables"=>{
+                                "length"=>141,
+                                "tos"=>"00", "precedence_bits"=>"00",
+                                "ttl"=>63,
+                                "id"=>"55251"
+                            },
+                            "network"=>{"transport"=>"UDP"},
+                            "source"=>{"ip"=>"1.2.3.4", "port"=>5353},
+                            "destination"=>{"ip"=>"192.168.0.10", "port"=>5335}
+                        )
+      else
+        expect(subject).to include("nf_host" => "myHost")
+        expect(subject).to include("nf_action1" => "FORWARD")
+        expect(subject).to include("nf_action2" => "REJECT")
+        expect(subject).to include("nf_in_interface" => "eth2")
+        expect(subject).to include("nf_out_interface" => "eth2")
+        expect(subject).to include("nf_src_ip" => "1.2.3.4")
+        expect(subject).to include("nf_dst_ip" => "192.168.0.10")
+        expect(subject).to include("nf_len" => "141")
+        expect(subject).to include("nf_tos" => "0x00")
+        expect(subject).to include("nf_prec" => "0x00")
+        expect(subject).to include("nf_ttl" => "63")
+        expect(subject).to include("nf_id" => "55251")
+        expect(subject).to include("nf_protocol" => "UDP")
+        expect(subject).to include("nf_src_port" => "5353")
+        expect(subject).to include("nf_dst_port" => "5335")
+      end
+    end
 
-    it { should include("nf_dst_port" => "5353") }
   end
 
   context "parsing a message without OUT interface" do
 
-    let(:value) { "May 28 17:31:07 myHost kernel: [3125121.106700] Shorewall:net2fw:DROP:IN=eth1 OUT= MAC=00:02:b3:c7:2f:77:38:72:c0:6e:92:9c:08:00 SRC=1.2.3.4 DST=1.2.3.4 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=6480 DF PROTO=TCP SPT=59088 DPT=8080 WINDOW=2920 RES=0x00 SYN URGP=0" }
-
-    subject     { grok_match(pattern, value) }
-
-    it { should include("timestamp" => "May 28 17:31:07") }
-
-    it { should include("nf_host" => "myHost") }
-
-    it { should include("nf_action1" => "net2fw") }
-
-    it { should include("nf_action2" => "DROP") }
-
-    it { should include("nf_in_interface" => "eth1") }
-
-    it { expect(subject["nf_out_interface"]).to be_nil }
-
-    it { should include("nf_dst_mac" => "00:02:b3:c7:2f:77") }
-
-    it { should include("nf_src_mac" => "38:72:c0:6e:92:9c") }
-
-    it { should include("nf_src_ip" => "1.2.3.4") }
-
-    it { should include("nf_dst_ip" => "1.2.3.4") }
-
-    it { should include("nf_len" => "60") }
-
-    it { should include("nf_tos" => "0x00") }
-
-    it { should include("nf_prec" => "0x00") }
-
-    it { should include("nf_ttl" => "49") }
-
-    it { should include("nf_id" => "6480") }
-
-    it { should include("nf_protocol" => "TCP") }
 
-    it { should include("nf_src_port" => "59088") }
+    let(:message) do
+      "May 28 17:31:07 server Shorewall:net2fw:DROP:" +
+      "IN=eth1 OUT= MAC=00:02:b3:c7:2f:77:38:72:c0:6e:92:9c:08:00 SRC=127.0.0.1 DST=1.2.3.4 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=6480 DF PROTO=TCP SPT=59088 DPT=8080 WINDOW=2920 RES=0x00 SYN URGP=0"
+    end
+
+    it 'matches' do
+      expect(subject).to include("timestamp" => "May 28 17:31:07")
+      if ecs_compatibility?
+        expect(subject).to include(
+                            "observer"=>{"hostname"=>"server", "ingress"=>{"interface"=>{"name"=>"eth1"}}}, # no "output_interface"
+                            "shorewall"=>{'firewall'=>{"type"=>"net2fw", "action"=>"DROP",}},
+                            "iptables"=>{
+                                "length"=>60,
+                                "tos"=>"00", "precedence_bits"=>"00",
+                                "ttl"=>49,
+                                "id"=>"6480",
+
+                                "fragment_flags"=>"DF",
+                                "tcp"=>{"flags"=>"SYN ", "window"=>2920},
+                                "tcp_reserved_bits"=>"00",
+                            },
+                            "network"=>{"transport"=>"TCP"}
+                        )
+        expect(subject).to include("source"=>{"ip"=>"127.0.0.1", "port"=>59088, 'mac'=>"38:72:c0:6e:92:9c"})
+        expect(subject).to include("destination"=>{"ip"=>"1.2.3.4", "port"=>8080, 'mac'=>"00:02:b3:c7:2f:77"})
+      else
+        expect(subject).to include("nf_host" => "server")
+        expect(subject).to include("nf_action1" => "net2fw")
+        expect(subject).to include("nf_action2" => "DROP")
+        expect(subject).to include("nf_in_interface" => "eth1")
+        expect(subject["nf_out_interface"]).to be nil
+        expect(subject).to include("nf_dst_mac" => "00:02:b3:c7:2f:77")
+        expect(subject).to include("nf_src_mac" => "38:72:c0:6e:92:9c")
+        expect(subject).to include("nf_src_ip" => "127.0.0.1")
+        expect(subject).to include("nf_dst_ip" => "1.2.3.4")
+        expect(subject).to include("nf_len" => "60")
+        expect(subject).to include("nf_tos" => "0x00")
+        expect(subject).to include("nf_prec" => "0x00")
+        expect(subject).to include("nf_ttl" => "49")
+        expect(subject).to include("nf_id" => "6480")
+        expect(subject).to include("nf_protocol" => "TCP")
+        expect(subject).to include("nf_src_port" => "59088")
+        expect(subject).to include("nf_dst_port" => "8080")
+      end
+    end
 
-    it { should include("nf_dst_port" => "8080") }
   end
 end

data/spec/patterns/squid_spec.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe_pattern "SQUID3", ['legacy', 'ecs-v1'] do
+
+  describe 'CONNECT sample' do
+
+    let(:message) do
+      '1525344856.899  16867 10.170.72.111 TCP_TUNNEL/200 6256 CONNECT logs.ap-southeast-2.amazonaws.com:443 - HIER_DIRECT/53.140.206.134 -'
+    end
+
+    it "matches" do
+      expect(subject).to include("timestamp" => "1525344856.899")
+      if ecs_compatibility?
+        expect(subject).to include(
+                            "event" => { "action" => "TCP_TUNNEL" },
+                            "squid" => {
+                                "request" => { "duration" => 16867 },
+                                "hierarchy_code" => "HIER_DIRECT"
+                            })
+        expect(subject).to include("destination" => { "address" => "53.140.206.134" })
+        expect(subject).to include("http" => {
+            "request" => { "method" => "CONNECT" },
+            "response" => { "bytes" => 6256, "status_code" => 200 }
+            # does not include missing ('-') as http.response.mime_type
+        })
+        expect(subject).to include("url" => { "original" => "logs.ap-southeast-2.amazonaws.com:443" })
+        expect(subject).to include("source" => { "ip" => "10.170.72.111" })
+      else
+        expect(subject).to include(
+                            "duration" => "16867",
+                            "client_address" => "10.170.72.111",
+                            "cache_result" => "TCP_TUNNEL",
+                            "status_code" => "200",
+                            "request_method" => "CONNECT",
+                            "bytes" => "6256",
+                            "url" => "logs.ap-southeast-2.amazonaws.com:443",
+                            "user" => "-",
+                            "hierarchy_code" => "HIER_DIRECT",
+                            "server" => "53.140.206.134",
+                            "content_type" => "-",
+                            )
+      end
+    end
+
+    it "does not include missing ('-') user-name" do
+      if ecs_compatibility?
+        expect(subject.keys).to_not include("user") # "user" => { "name" => "-" }
+      end
+    end
+
+  end
+
+  describe 'GET sample' do
+
+    let(:message) do
+      "1525334330.556      3 120.65.1.1 TCP_REFRESH_MISS/200 2014 GET http://www.sample.com/hellow_world.txt public-user DIRECT/www.sample.com text/plain 902351708.872"
+    end
+
+    it "matches" do
+      expect(subject).to include("timestamp" => "1525334330.556")
+      if ecs_compatibility?
+        expect(subject).to include(
+                            "event" => { "action" => "TCP_REFRESH_MISS" },
+                            "squid" => {
+                                "request" => { "duration" => 3 },
+                                "hierarchy_code" => "DIRECT"
+                            })
+        expect(subject).to include("destination" => { "address" => "www.sample.com" })
+        expect(subject).to include("http" => {
+            "request" => { "method" => "GET" },
+            "response" => { "bytes" => 2014, "status_code" => 200, "mime_type" => 'text/plain' }
+        })
+        expect(subject).to include("url" => { "original" => "http://www.sample.com/hellow_world.txt" })
+        expect(subject).to include("source" => { "ip" => "120.65.1.1" })
+        expect(subject).to include("user" => { "name" => "public-user" })
+      else
+        expect(subject).to include(
+                            "duration"=>"3",
+                            "client_address"=>"120.65.1.1",
+                            "cache_result"=>"TCP_REFRESH_MISS",
+                            "status_code"=>"200",
+                            "bytes"=>"2014",
+                            "request_method" => "GET",
+                            "url" => "http://www.sample.com/hellow_world.txt",
+                            "user"=>"public-user",
+                            "hierarchy_code"=>"DIRECT",
+                            "server"=>"www.sample.com",
+                            "content_type"=>"text/plain",
+                            )
+      end
+    end
+
+    it "retains message" do
+      expect(subject).to include("message" => message)
+    end
+
+  end
+
+  context 'GET with invalid status' do # code 0 means unavailable - no response received
+
+    let(:message) do
+      '1426235912.366     16 192.168.3.50 TCP_MISS_ABORTED/000 0 GET http://garfield.com/comic/2015-03-13 - HIER_DIRECT/193.135.59.44 -'
+    end
+
+    it "matches" do
+      expect(subject).to include("timestamp" => "1426235912.366")
+      if ecs_compatibility?
+        expect(subject).to include("event"=>{"action"=>"TCP_MISS_ABORTED"},
+                                   "http"=>{"response"=>{"bytes"=>0}, "request"=>{"method"=>"GET"}})
+      else
+        expect(subject).to include("status_code"=>'000')
+      end
+    end
+
+  end
+
+  context 'GET with optional server ip' do
+
+    let(:message) do
+      '1066037222.352     132 144.157.100.17 TCP_MISS/504 1293 GET http://at.atremis.com/image/93101912/xyz - NONE/- -'
+    end
+
+    it "matches" do
+      expect(subject).to include("timestamp" => "1066037222.352")
+      if ecs_compatibility?
+        expect(subject).to include("event"=>{"action"=>"TCP_MISS"},
+                                   "http"=>{"response"=>{"bytes"=>1293, "status_code"=>504}, "request"=>{"method"=>"GET"}})
+        expect(subject.keys).not_to include('destination')
+      else
+        expect(subject).to include("status_code"=>'504')
+        expect(subject.keys).not_to include('server')
+      end
+    end
+
+  end
+
+end

data/spec/patterns/syslog_spec.rb

@@ -2,48 +2,292 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
-describe "SYSLOGLINE" do
-
-  describe "SYSLOG5424BASE" do
-     it "matches host names in the syslog base pattern" do
-        expect(subject).to match("<174>1 2016-11-14T09:32:44+01:00 resolver.se named 6344 - -  info: client 10.23.53.22#63252: query: googlehosted.l.googleusercontent.com IN A + (10.23.16.6)")
-     end
-     
-     it "matches ipv4 in the syslog base pattern" do
-        expect(subject).to match("<174>1 2016-11-14T09:49:23+01:00 10.23.16.6 named 2255 - -  info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)")
-     end
-
-     it "matches ipv6 in the syslog base pattern" do
-        expect(subject).to match("<174>1 2016-11-14T09:49:23+01:00 2000:6a0:b:315:10:23:4:13 named 2255 - -  info: client 10.23.56.9#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)")
-     end
-   end
+describe_pattern "SYSLOGLINE", ['legacy', 'ecs-v1'] do
 
   it "matches a simple message with pid" do
-    expect(subject).to match("May 11 15:17:02 meow.soy.se CRON[10973]: pam_unix(cron:session): session opened for user root by (uid=0)")
+    match = grok_match pattern, "May 11 15:17:02 meow.soy.se CRON[10973]: pam_unix(cron:session): session opened for user root by (uid=0)"
+    if ecs_compatibility?
+      expect(match).to include("process" => { "name" => "CRON", "pid" => 10973 })
+    else
+      expect(match).to include("pid" => "10973", "program" => "CRON")
+    end
   end
 
   it "matches prog with slash" do
-    expect(subject).to match("Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]")
+    match = grok_match pattern, "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
+    if ecs_compatibility?
+      expect(match).to include("process" => { "name" => "postfix/smtpd", "pid" => 1713 })
+    else
+      expect(match).to include("program" => "postfix/smtpd")
+    end
   end
 
   it "matches prog from ansible" do
-    expect(subject).to match("May 11 15:40:51 meow.soy.se ansible-<stdin>: Invoked with filter=* fact_path=/etc/ansible/facts.d")
+    message = "May 11 15:40:51 meow.soy.se ansible-<stdin>: Invoked with filter=* fact_path=/etc/ansible/facts.d"
+    match = grok_match pattern, message
+    if ecs_compatibility?
+      expect(match).to include(
+                           "timestamp" => "May 11 15:40:51",
+                           "process" => { "name" => "ansible-<stdin>" },
+                           "host" => { "hostname" => "meow.soy.se" },
+                           "message" => [message, "Invoked with filter=* fact_path=/etc/ansible/facts.d"]
+                       )
+    else
+      expect(match).to include(
+                           "timestamp" => "May 11 15:40:51",
+                           "logsource" => "meow.soy.se",
+                           "message" => [message, "Invoked with filter=* fact_path=/etc/ansible/facts.d"]
+                       )
+    end
   end
 
   it "matches prog from RFC5424 APP-NAME" do
     # https://tools.ietf.org/html/rfc5424#section-6.2.5
     # https://tools.ietf.org/html/rfc5424#section-6
     tag_from_rfc = ((33..126).map { |c| c.chr } - %w{[ ]}).join
-    expect(subject).to match("May 11 15:40:51 meow.soy.se #{tag_from_rfc}: Just some data which conforms to RFC5424")
+    match = grok_match pattern, "May 11 15:40:51 meow.soy.se #{tag_from_rfc}: Just some data which conforms to RFC5424"
+    if ecs_compatibility?
+      expect(match).to include("process" => { "name" => tag_from_rfc })
+      expect(match).to include("host" => { "hostname" => "meow.soy.se" })
+    else
+      expect(match).to include("logsource" => "meow.soy.se", "program" => tag_from_rfc)
+    end
+  end
+
+  it 'does not parse facility-level or msg-id' do
+    message = 'May 11 10:40:48 scrooge disk-health-nurse[26783]: [ID 702911 user.error] m:SY-mon-full-500 c:H : partition health measures for /var did not suffice - still using 96% of partition space'
+    match = grok_match pattern, message
+    expect(match).to include("timestamp" => "May 11 10:40:48")
+    expect(match).to include("message" => [message, "[ID 702911 user.error] m:SY-mon-full-500 c:H : partition health measures for /var did not suffice - still using 96% of partition space"])
+    if ecs_compatibility?
+      expect(match).to include("process"=>{"pid"=>26783, "name"=>"disk-health-nurse"}, "host"=>{"hostname"=>"scrooge"})
+    else
+      expect(match).to include("program"=>"disk-health-nurse", "pid"=>"26783", "logsource"=>"scrooge")
+    end
+  end
+
+  it 'parses (non-syslog) mesages without hostname' do
+    message = "Jan 11 22:33:44 su: 'su root' failed for luser on /dev/pts/8"
+    match = grok_match pattern, message
+    if ecs_compatibility? # in legacy mode a parse failure
+      expect(match).to include("process" => { "name" => "su" })
+    end
   end
 
   context "when having an optional progname" do
 
-    let(:pattern) { "SYSLOGLINE" }
-    let(:value)   { "<14>Jun 24 10:32:02 hostname WinFileService Event: read, Path: /.DS_Store, File/Folder: File, Size: 6.00 KB, User: user@host, IP: 123.123.123.123" }
+    let(:message) { "<14>Jun 24 10:32:02 win-host WinFileService Event: read, Path: /.DS_Store, File/Folder: File, Size: 6.00 KB, User: user@host, IP: 123.123.123.123" }
 
     it "should accept the message" do
-      expect(grok_match(pattern, value)).to pass
+      if ecs_compatibility?
+        expect(grok).to include("host" => { "hostname" => "win-host" })
+      else
+        expect(grok).to include("logsource" => "win-host")
+      end
+      expect(grok['message']).to eql [message, 'WinFileService Event: read, Path: /.DS_Store, File/Folder: File, Size: 6.00 KB, User: user@host, IP: 123.123.123.123']
+    end
+  end
+end
+
+describe_pattern "SYSLOG5424LINE", ['legacy', 'ecs-v1'] do
+
+  it "matches (ipv4 host)" do
+    message = "<174>1 2016-11-14T09:49:23+01:00 10.23.16.6 named 2255 - -  info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"
+    match = grok_match pattern, message
+    if ecs_compatibility?
+      expect(match).to include("log" => { "syslog" => { "facility" => { "code" => 174 }}})
+      expect(match).to include("host" => { "hostname" => "10.23.16.6"})
+      expect(match).to include("process" => { "name" => "named", "pid" => 2255 })
+      expect(match).to include("system" => { "syslog" => { "version" => "1" }})
+      expect(match).to include("timestamp" => "2016-11-14T09:49:23+01:00")
+      expect(match).to include("message" => [message, "info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"])
+    else
+      expect(match).to include({
+                                   "syslog5424_pri" => "174",
+                                   "syslog5424_host" => "10.23.16.6",
+                                   "syslog5424_app" => "named",
+                                   "syslog5424_ver" => "1",
+                                   "syslog5424_proc" => "2255",
+                                   "syslog5424_ts" => "2016-11-14T09:49:23+01:00",
+                                   "syslog5424_msg" => "info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"
+                               })
+      expect(match).to include("message" => message)
+    end
+  end
+
+  it "matches ipv6 host" do
+    message = "<174>1 1995-04-12T23:20:50.52Z 2000:6a0:b:315:10:23:4:13 named 2255 - -  info: client 10.23.56.9#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"
+    match = grok_match pattern, message
+    if ecs_compatibility?
+      expect(match).to include("host" => { "hostname" => "2000:6a0:b:315:10:23:4:13" })
+      expect(match).to include("timestamp" => "1995-04-12T23:20:50.52Z")
+      expect(match).to include("message" => [message, "info: client 10.23.56.9#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"])
+    else
+      expect(match).to include("syslog5424_host" => "2000:6a0:b:315:10:23:4:13")
+      expect(match).to include("syslog5424_ts" => "1995-04-12T23:20:50.52Z")
+    end
+  end
+
+  it "matches host name" do
+    message = "<174>1 2016-12-31T23:59:60-04:00 resolver.se prg00000[1234] - -  info: client 10.23.53.22#63252: query: googlehosted.l.googleusercontent.com IN A + (10.23.16.6)"
+    match = grok_match pattern, message
+    if ecs_compatibility?
+      expect(match).to include("host" => { "hostname" => "resolver.se" })
+      expect(match).to include("message" => [message, "info: client 10.23.53.22#63252: query: googlehosted.l.googleusercontent.com IN A + (10.23.16.6)"])
+      expect(match).to include("timestamp" => "2016-12-31T23:59:60-04:00")
+      expect(match['system']['syslog'].keys).to_not include("message_id")
+    else
+      expect(match).to include("syslog5424_host" => "resolver.se")
+      expect(match).to include("syslog5424_ts" => "2016-12-31T23:59:60-04:00")
+      expect(match.keys).to_not include("syslog5424_msgid")
+    end
+  end
+
+  it "matches nil host-name" do
+    message = "<174>1 2003-08-24T05:14:15+00:00 - - - - - IN=enx503f560021dc OUT= MAC=01:00:5e:00:00:01:88:f8:c7:8b:1a:b4:04:00 SRC=192.168.81.1 DST=224.0.0.1 LEN=32 TOS=0x00 PREC=0xC0 TTL=1 ID=0 PROTO=2"
+    match = grok_match pattern, message
+    if ecs_compatibility?
+      expect(match).to include("timestamp" => "2003-08-24T05:14:15+00:00")
+      expect(match.keys).to_not include("host")
+    else
+      expect(match.keys).to_not include("syslog5424_host")
+    end
+  end
+
+  it 'matches milli-second precision timestamp' do
+    match = grok_match pattern, "<34>1 2003-08-24T05:14:15.123-07:00 the.borg.com su - ID47 - BOM'su root' failed for borg on /dev/pts/7"
+    if ecs_compatibility?
+      expect(match).to include("timestamp" => "2003-08-24T05:14:15.123-07:00")
+      expect(match).to include("process" => { "name" => "su" })
+    else
+      expect(match).to include("syslog5424_ts" => "2003-08-24T05:14:15.123-07:00")
+    end
+  end
+
+  it 'matches milli-second precision timestamp (Z) and msgid' do
+    match = grok_match pattern, "<34>1 2030-10-11T22:14:15.003Z the.borg.com su - m:WR-ID47 - BOM'su root' failed for borg on /dev/pts/7"
+    if ecs_compatibility?
+      expect(match).to include("timestamp" => "2030-10-11T22:14:15.003Z")
+      expect(match).to include("event" => { "code" => 'm:WR-ID47' })
+    else
+      expect(match).to include("syslog5424_ts" => "2030-10-11T22:14:15.003Z")
+      expect(match).to include("syslog5424_msgid" => "m:WR-ID47")
+    end
+  end
+
+  it 'matches micro-second precision timestamp and structured-data' do
+    message = "<34>1 2012-12-19T01:45:54.001234Z rufus - - - [meta sequenceId=10] BOMStack unit 3 Power supply 2 is down"
+    match = grok_match pattern, message
+    if ecs_compatibility?
+      expect(match).to include("timestamp" => "2012-12-19T01:45:54.001234Z")
+      expect(match).to include("system" => { "syslog" =>  hash_including("structured_data" => '[meta sequenceId=10]') })
+      expect(match).to include("message" => [message, "BOMStack unit 3 Power supply 2 is down"])
+
+      expect(match.keys).to_not include("process") # process.name
+      expect(match.keys).to_not include("event") # event.code
+    else
+      expect(match).to include("syslog5424_ts" => "2012-12-19T01:45:54.001234Z")
+      expect(match).to include("syslog5424_sd" => "[meta sequenceId=10]")
+      expect(match).to include("syslog5424_msg" => "BOMStack unit 3 Power supply 2 is down")
+    end
+  end
+
+  it 'matches more structured-data' do
+    sd = '[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][examplePriority@32473 class="high"]'
+    message = "<34>1 2012-12-19T01:45:54.001234+11:30 rufus - - - #{sd} [down]"
+    match = grok_match pattern, message
+    if ecs_compatibility?
+      expect(match).to include("system" => { "syslog" =>  hash_including("structured_data" => sd) })
+      expect(match).to include("message" => [message, "[down]"])
+    else
+      expect(match).to include("syslog5424_sd" => sd)
+      expect(match).to include("syslog5424_msg" => "[down]")
+    end
+  end
+
+end
+
+describe_pattern 'SYSLOGPAMSESSION', ['legacy', 'ecs-v1'] do
+
+  it "matches" do
+    message = 'Jul 14 13:36:03 precision pkexec: pam_unix(polkit-1:session): session opened for user root by (uid=1001)'
+    match = grok_match pattern, message
+    expect(match).to include("timestamp" => "Jul 14 13:36:03")
+    if ecs_compatibility?
+      expect(match).to include(
+                           "host" => { "hostname" => "precision" },
+                           "process" => { "name" => "pkexec" },
+                           "user" => { "name" => "root" },
+                           "system" => { "auth" => {
+                               "pam" => { "module" => "pam_unix", "origin" => "polkit-1:session", "session_state" => "opened" }}
+                           }
+                       )
+    else
+      expect(match).to include({
+                                   "logsource"=>"precision",
+                                   "program"=>"pkexec",
+                                   "username"=>"root",
+                                   "pam_module"=>"pam_unix",
+                                   "pam_caller"=>"polkit-1:session",
+                                   "pam_session_state"=>"opened",
+                                   "pam_by"=>"(uid=1001)",
+                               })
+    end
+    expect(match).to include("message" => [message, "pam_unix(polkit-1:session): session opened for user root by (uid=1001)"])
+  end
+
+  it "matches a message with pid" do
+    message = 'Jul 14 12:17:01.234 precision.computer CRON[869567]: pam_unix(cron:session): session closed for user root'
+    match = grok_match pattern, message
+    expect(match).to include("timestamp" => "Jul 14 12:17:01.234")
+    if ecs_compatibility?
+      expect(match).to include(
+                           "host" => { "hostname" => "precision.computer" },
+                           "process" => { "name" => "CRON", "pid" => 869567 },
+                           "user" => { "name" => "root" },
+                           "system" => { "auth" => {
+                               "pam" => { "module" => "pam_unix", "origin" => "cron:session", "session_state" => "closed" }}
+                           }
+                       )
+    else
+      expect(match).to include({
+                                   "logsource" => "precision.computer",
+                                   "program" => "CRON",
+                                   "username" => "root",
+                                   "pid" => "869567",
+                                   "pam_module" => "pam_unix",
+                                   "pam_caller" => "cron:session",
+                                   "pam_session_state" => "closed"
+                               })
     end
+    expect(match).to include("message" => [message, "pam_unix(cron:session): session closed for user root"])
   end
+
 end
+
+describe_pattern 'CRONLOG', ['legacy', 'ecs-v1'] do
+
+  it "matches" do
+    message = 'Jul 17 12:17:01 precision CRON[869568]: (root) CMD (   cd / && run-parts --report /etc/cron.hourly)'
+    match = grok_match pattern, message
+    expect(match).to include("timestamp" => "Jul 17 12:17:01")
+    if ecs_compatibility?
+      expect(match).to include(
+                           "host" => { "hostname" => "precision" },
+                           "process" => { "name" => "CRON", "pid" => 869568 },
+                           "user" => { "name" => "root" },
+                           "system" => { "cron" => { "action" => "CMD" } }
+                       )
+    else
+      expect(match).to include(
+                           "logsource"=>"precision",
+                           "program"=>"CRON",
+                           "pid"=>"869568",
+                           "user"=>"root",
+                           "action"=>"CMD"
+                       )
+    end
+    expect(match).to include("message"=>[message, "   cd / && run-parts --report /etc/cron.hourly"])
+  end
+
+end