logstash-patterns-core 4.1.2 → 4.3.2

data/spec/patterns/core_spec.rb

@@ -2,20 +2,20 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
-describe "SYSLOGLINE" do
+describe_pattern "SYSLOGLINE", ['legacy', 'ecs-v1'] do
+
+  let(:message) { "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]" }
 
-  let(:value)   { "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]" }
-  let(:grok)    { grok_match(subject, value) }
   it "a pattern pass the grok expression" do
     expect(grok).to pass
   end
 
-  it "matches a simple message" do
-    expect(subject).to match(value)
-  end
-
   it "generates the program field" do
-    expect(grok_match(subject, value)).to include("program" => "postfix/smtpd")
+    if ecs_compatibility?
+      expect(grok).to include("process" => hash_including('name' => 'postfix/smtpd'))
+    else
+      expect(grok).to include("program" => "postfix/smtpd")
+    end
   end
 
 end
@@ -54,12 +54,16 @@ describe "HTTP DATE parsing" do
 
 end
 
-describe "TOMCATLOG" do
-
-  let(:value) { '2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened...'}
+describe 'LOGLEVEL' do
+  it 'matches info label' do
+    expect(grok_match(subject, 'INFO')).to pass
+    expect(grok_match(subject, 'info')).to pass
+  end
 
-  it "generates the logmessage field" do
-    expect(grok_match(subject, value)).to include("logmessage" => "something compeletely unexpected happened...")
+  it 'matches information label' do
+    expect(grok_match(subject, 'information')).to pass
+    expect(grok_match(subject, 'Information')).to pass
+    expect(grok_match(subject, 'INFORMATION')).to pass
   end
 end
 
@@ -90,19 +94,209 @@ describe "UNIXPATH" do
   let(:value)   { '/foo/bar' }
 
   it "should match the path" do
-    expect(grok_match(pattern,value)).to pass
+    expect(grok_match(pattern, value, true)).to pass
   end
 
   context "when using comma separators and other regexp" do
 
+    let(:pattern) { '((a=(?<a>%{UNIXPATH})?|b=(?<b>%{UNIXPATH})?)(,\s)?)+' }
+
+    let(:grok) do
+      grok = LogStash::Filters::Grok.new("match" => ["message", pattern])
+      grok.register
+      grok
+    end
+
     let(:value) { 'a=/some/path, b=/some/other/path' }
 
+    it "was expected to extract both but never really did" do # or maybe on JRuby 1.7
+      event = build_event(value)
+      grok.filter(event)
+      expect( event.to_hash['a'] ).to eql '/some/path,'
+      expect( event.to_hash['b'] ).to be nil
+    end
+
+  end
+
+  context 'relative path' do
+
+    let(:path_matcher) do # non-exact matcher
+      grok = LogStash::Filters::Grok.new("match" => ["message", '%{UNIXPATH:path}'])
+      grok.register
+      lambda { |msg| event = build_event(msg); grok.filter(event); event }
+    end
+
+    it "should not match (only partially)" do
+      expect(grok_match(pattern, 'a/./b/c', true)).to_not pass
+      event = path_matcher.('a/./b/c')
+      expect( event.to_hash['path'] ).to eql '/./b/c'
+
+      expect(grok_match(pattern, ',/.', true)).to_not pass
+      event = path_matcher.(',/.')
+      expect( event.to_hash['path'] ).to eql '/.'
+
+      expect(grok_match(pattern, '+/.../', true)).to_not pass
+      event = path_matcher.('+/.../')
+      expect( event.to_hash['path'] ).to eql '/.../'
+
+      expect(grok_match(pattern, '~/b/', true)).to_not pass
+      event = path_matcher.('~/b/')
+      expect( event.to_hash['path'] ).to eql '/b/'
+
+      expect(grok_match(pattern, './b//', true)).to_not pass
+      expect(grok_match(pattern, 'a//b', true)).to_not pass
+    end
+
+    it "should not match paths starting with ." do
+      expect(grok_match(pattern, '../0', true)).to_not pass
+      expect(grok_match(pattern, './~', true)).to_not pass
+      expect(grok_match(pattern, '.../-', true)).to_not pass
+      expect(grok_match(pattern, './', true)).to_not pass
+      expect(grok_match(pattern, './,', true)).to_not pass
+      expect(grok_match(pattern, '../', true)).to_not pass
+      expect(grok_match(pattern, '.a/', true)).to_not pass
+      expect(grok_match(pattern, '.~/', true)).to_not pass
+    end
+
+    it "should not match expression wout separator" do
+      expect(grok_match(pattern, '.')).to_not pass
+      expect(grok_match(pattern, '..')).to_not pass
+      expect(grok_match(pattern, '...')).to_not pass
+      expect(grok_match(pattern, '.,')).to_not pass
+      expect(grok_match(pattern, '.-')).to_not pass
+    end
+
+  end
+
+  context "dotted path" do
+
+    it "should match path containing ." do
+      expect(grok_match(pattern, '/some/./path/', true)).to pass
+      expect(grok_match(pattern, '/some/../path', true)).to pass
+      expect(grok_match(pattern, '/../.', true)).to pass
+      expect(grok_match(pattern, '/.', true)).to pass
+      expect(grok_match(pattern, '/..', true)).to pass
+      expect(grok_match(pattern, '/...', true)).to pass
+    end
+
+  end
+
+  context "separators" do
+
+    it "should match root" do
+      expect(grok_match(pattern, '/', true)).to pass
+    end
+
+    it "should match" do
+      expect(grok_match(pattern, '//', true)).to pass
+      expect(grok_match(pattern, '//00', true)).to pass
+      expect(grok_match(pattern, '///a', true)).to pass
+      expect(grok_match(pattern, '/a//', true)).to pass
+      expect(grok_match(pattern, '///a//b/c///', true)).to pass
+    end
+
+    it "should not match windows separator" do
+      expect(grok_match(pattern, "\\a", true)).to_not pass
+      expect(grok_match(pattern, '/0\\', true)).to_not pass
+      expect(grok_match(pattern, "/a\\b", true)).to_not pass
+    end
+
+  end
+
+  context "long path" do
+
+    let(:grok) do
+      grok = LogStash::Filters::Grok.new("match" => ["message", '%{UNIXPATH:path} '], 'timeout_millis' => 1500)
+      grok.register
+      grok
+    end
+
+    let(:value) { '/opt/abcdef/1/.22/3:3+3/foo@BAR/X-Y+Z/~Sample_l_SUBc b' }
+
     it "should match the path" do
-      expect(grok_match(pattern,value)).to pass
+      event = build_event(value)
+      grok.filter(event)
+      expect( event.to_hash['path'] ).to eql '/opt/abcdef/1/.22/3:3+3/foo@BAR/X-Y+Z/~Sample_l_SUBc'
     end
+
+    it "should not match with invalid chars (or cause DoS)" do
+      event = build_event(value.sub('SUB', '&^_'))
+      grok.filter(event) # used to call a looong looop (DoS) despite the timeout guard
+      expect( event.to_hash['tags'] ).to include '_grokparsefailure'
+    end
+  end
+
+  it "matches paths with non-ascii characters" do
+    event = build_event path = '/opt/Čierný_Peter/.中'
+    build_grok('UNIXPATH:path').filter event
+    expect( event.get('path') ).to eql path
+  end
+
+end
+
+describe "WINPATH" do
+
+  let(:pattern) { 'WINPATH' }
+  let(:value)   { 'C:\\foo\\bar' }
+
+  it "should match the path" do
+    expect(grok_match(pattern, value, true)).to pass
+  end
+
+  it "should match root path" do
+    expect(grok_match(pattern, 'C:\\', true)).to pass
+    expect(grok_match(pattern, 'C:\\\\', true)).to pass
+    expect(grok_match(pattern, 'a:\\', true)).to pass
+    expect(grok_match(pattern, 'x:\\\\', true)).to pass
+  end
+
+  it "should match paths with spaces" do
+    expect(grok_match(pattern, 'C:\\Documents and Settings\\Public', true)).to pass
+    expect(grok_match(pattern, 'C:\\\\Users\\\\Public\\\\.Mozilla Firefox', true)).to pass
+  end
+
+  it "should not match unix-style paths" do
+    expect(grok_match(pattern, '/foo', true)).to_not pass
+    expect(grok_match(pattern, '//C/path', true)).to_not pass
+    expect(grok_match(pattern, '/', true)).to_not pass
+    expect(grok_match(pattern, '/foo/bar', true)).to_not pass
+    expect(grok_match(pattern, '/..', true)).to_not pass
+    expect(grok_match(pattern, 'C://', true)).to_not pass
+  end
+
+  it "matches paths with non-ascii characters" do
+    expect(grok_match(pattern, 'C:\\Čierný Peter\\.中.exe', true)).to pass
+  end
+
+  context 'relative paths' do
+
+    it "should not match" do
+      expect(grok_match(pattern, 'a\\bar', true)).to_not pass
+      expect(grok_match(pattern, 'foo\\bar', true)).to_not pass
+      expect(grok_match(pattern, 'C\\A\\B', true)).to_not pass
+      expect(grok_match(pattern, 'C\\\\0', true)).to_not pass
+      expect(grok_match(pattern, '.\\0', true)).to_not pass
+      expect(grok_match(pattern, '..\\', true)).to_not pass
+      expect(grok_match(pattern, '...\\-', true)).to_not pass
+      expect(grok_match(pattern, '.\\', true)).to_not pass
+      expect(grok_match(pattern, '.\\,', true)).to_not pass
+      expect(grok_match(pattern, '..\\', true)).to_not pass
+      expect(grok_match(pattern, '.a\\', true)).to_not pass
+    end
+
+    it "should not match expression wout separator" do
+      expect(grok_match(pattern, '.')).to_not pass
+      expect(grok_match(pattern, '..')).to_not pass
+      expect(grok_match(pattern, '...')).to_not pass
+      expect(grok_match(pattern, 'C:')).to_not pass
+      expect(grok_match(pattern, 'C')).to_not pass
+    end
+
   end
+
 end
 
+
 describe "URIPROTO" do
   let(:pattern) { 'URIPROTO' }
 
@@ -292,3 +486,54 @@ describe "URN" do
     end
   end
 end
+
+describe_pattern "EMAILADDRESS", ['legacy', 'ecs-v1'] do
+
+  # NOTE: EMAILLOCALPART was only updated in ECS mode, as following the RFC is
+  # actually a breaking change -> legacy does match incorrect e-mail addresses.
+
+  it "matches 'hello.world@123.net' address" do
+    expect(grok_exact_match(pattern, 'hello.world@123.net')).to pass
+  end
+
+  [
+      'a@example.com',
+      'a{b}@example.com',
+      'Foo+Bar@x.exposed',
+  ].each do |valid_email|
+    it "matches #{valid_email.inspect} address" do
+      expect(grok_exact_match(pattern, valid_email)).to pass if ecs_compatibility?
+    end
+  end
+
+  it "does not match 'x y@example.com' address" do
+    expect(grok_exact_match(pattern, 'hello.world@123.net')).to pass
+  end
+
+  [
+      'a:b@example.com',
+      'a..b@example.com',
+  ].each do |invalid_email|
+    it "does not match #{invalid_email.inspect} address" do
+      expect(grok_exact_match(pattern, invalid_email)).to_not pass if ecs_compatibility?
+    end
+  end
+
+  it "matches e-mail with digits only local-part" do
+    expect(grok_exact_match(pattern, '00@q.ro')).to pass if ecs_compatibility?
+  end
+
+  it "matches e-mail with 64 chars in local-part" do
+    expect(grok_exact_match(pattern, ('ab' * 32) + '@root.cz')).to pass
+    expect(grok_exact_match(pattern, ('a.bc' * 16) + '@00.cz')).to pass
+  end
+
+  it "does not match e-mail with more than 64 char length local-part" do
+    if ecs_compatibility?
+      expect(grok_exact_match(pattern, ('ab' * 32) + 'a' + '@root.cz')).to_not pass
+      # NOTE: we allow longer with '.' but at least we limit "too long" :
+      expect(grok_exact_match(pattern, ('a.bc' * 64) + '@00.cz')).to_not pass
+    end
+  end
+
+end

data/spec/patterns/exim_spec.rb

@@ -0,0 +1,201 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe_pattern 'EXIM', ['legacy', 'ecs-v1'] do
+
+  context 'message arrival (old)' do
+
+    let(:message) do
+      "1995-10-31 08:57:53 0tACW1-0005MB-00 <= kryten@dwarf.fict.example H=mailer.fict.example [192.168.123.123] " +
+          "U=exim P=smtp S=5678 id=f828ca60127d8646a0fa75cbf8db9ba3@dwarf.fict.example"
+    end
+
+    it "matches" do
+      expect(grok).to include("timestamp" => "1995-10-31 08:57:53")
+
+      if ecs_compatibility?
+        expect(grok).to include("exim"=>{"log"=>{
+            "flags"=>"<=",
+            "sender"=>{"email"=>"kryten@dwarf.fict.example"},
+            "message"=>{"id"=>"0tACW1-0005MB-00", "size"=>5678},
+            "header_id"=>"f828ca60127d8646a0fa75cbf8db9ba3@dwarf.fict.example"
+        }})
+        expect(grok).to include("source"=>{"address"=>"mailer.fict.example", "ip"=>"192.168.123.123"})
+        expect(grok).to include("network"=>{"protocol"=>"smtp"})
+      else
+        expect(grok).to include("exim_year" => "1995", "exim_month" => "10", "exim_day" => "31", "@version" => "1", "exim_time" => "08:57:53")
+        expect(grok.keys).to_not include("pid")
+        expect(grok).to include("exim_sender_email" => "kryten@dwarf.fict.example")
+        expect(grok).to include("exim_flags" => "<=")
+        expect(grok).to include("exim_msg_size" => "5678")
+        expect(grok).to include("exim_msgid" => "0tACW1-0005MB-00")
+        expect(grok).to include("remote_hostname" => "mailer.fict.example", "remote_host" => "192.168.123.123")
+        expect(grok).to include("protocol" => "smtp")
+        expect(grok).to include("exim_header_id" => "f828ca60127d8646a0fa75cbf8db9ba3@dwarf.fict.example")
+      end
+
+      expect(grok).to include("message" => message)
+    end
+
+  end
+
+  context 'message arrival (new)' do
+    let(:message) do
+      '2010-09-13 05:00:13 [1487] 1Ov4tU-0000Nz-Rm <= mailling.list@domain.com ' +
+          'H=mailhost.domain.com [208.42.54.2]:51792 I=[67.215.162.175]:25 P=esmtps X=TLSv1:AES256-SHA:256 CV=no S=21778 ' +
+          'id=384a86a39e83be0d9b3a94d1feb3119f@domain.com T="Daily List: Chameleon" for user@example.com'
+    end
+
+    it "matches" do
+      expect(grok).to include("timestamp" => "2010-09-13 05:00:13") # new in legacy mode
+
+      if ecs_compatibility?
+        expect(grok).to include("process"=>{"pid"=>1487})
+        expect(grok).to include("exim"=>{"log"=>hash_including(
+            "message"=>{"id"=>"1Ov4tU-0000Nz-Rm", "size"=>21778, "subject"=>"Daily List: Chameleon"},
+            "flags"=>"<=",
+            "header_id"=>"384a86a39e83be0d9b3a94d1feb3119f@domain.com",
+            "sender"=>{"email"=>"mailling.list@domain.com"},
+            "recipient"=>{"email"=>"user@example.com"},
+        )})
+        expect(grok).to include("source"=>{"address"=>"mailhost.domain.com", "ip"=>"208.42.54.2", "port"=>51792})
+        expect(grok).to include("destination"=>{"ip"=>"67.215.162.175", "port"=>25})
+      else
+
+        expect(grok).to include("exim_year" => "2010", "exim_month" => "09", "exim_day" => "13", "exim_time" => "05:00:13")
+        expect(grok).to include("pid" => "1487") # new
+        expect(grok).to include("exim_sender_email" => "mailling.list@domain.com") # new
+        expect(grok).to include("remote_hostname" => "mailhost.domain.com", "remote_host" => "208.42.54.2", "remote_port" => "51792") # (remote_port) new
+        expect(grok).to include("exim_interface" => "67.215.162.175", "exim_interface_port" => "25")
+        expect(grok).to include("protocol" => "esmtps")
+        expect(grok).to include("exim_msg_size" => "21778")
+        expect(grok).to include("exim_header_id" => "384a86a39e83be0d9b3a94d1feb3119f@domain.com")
+        expect(grok).to include("exim_subject" => '"Daily List: Chameleon"')
+        expect(grok).to include("exim_recipient_email" => "user@example.com") # new
+      end
+
+      expect(grok).to include("message" => message)
+    end
+
+  end
+
+  context 'message arrival (simple)' do
+
+    let(:message) do
+      '2020-02-11 17:09:46 1j1Z2g-00Faoy-Uh <= example@strawberry.active-ns.com U=example P=local ' +
+          'T="[Examples Galore] Please moderate: \"Hello world!\"" for admin@example.net'
+    end
+
+    it "matches" do
+      expect(grok).to include("timestamp"=>"2020-02-11 17:09:46")
+      if ecs_compatibility?
+        expect(grok).to include("exim"=>{"log"=>{
+            "flags"=>"<=",
+            "message"=>{"id"=>"1j1Z2g-00Faoy-Uh", "subject"=>'[Examples Galore] Please moderate: \\"Hello world!\\"'},
+            "sender"=>{"email"=>"example@strawberry.active-ns.com"},
+            "recipient"=>{"email"=>"admin@example.net"}
+        }})
+        expect(grok).to include("network"=>{"protocol"=>"local"})
+      else
+        expect(grok).to include(
+                            "exim_msgid"=>"1j1Z2g-00Faoy-Uh",
+                            "exim_sender_email"=>"example@strawberry.active-ns.com",
+                            "exim_flags"=>"<=",
+                            "protocol"=>"local",
+                            "exim_subject"=>"\"[Examples Galore] Please moderate: \\\"Hello world!\\\"\""
+                        )
+      end
+    end
+
+  end
+
+  context 'message arrival with quoted hostname' do
+
+    let(:message) do
+      '2013-03-20 12:44:02 1UIIN7-0004t9-8R <= root@example.com ' +
+          'H=localhost (hostname.example.com) [127.0.0.1] ' +
+          'P=esmtps X=TLSv1:DHE-RSA-AES256-SHA:256 S=811 id=201303201244.r2KCi11V018784@hostname.example.com'
+    end
+
+    it "matches" do
+      expect(grok).to include("timestamp"=>"2013-03-20 12:44:02")
+
+      if ecs_compatibility?
+        expect(grok).to include("exim"=>{"log"=>hash_including(
+            "sender"=>{ "email"=>"root@example.com" },
+            "message"=>{ "id"=>"1UIIN7-0004t9-8R", "size"=>811 },
+            "header_id"=>"201303201244.r2KCi11V018784@hostname.example.com",
+            "remote_address"=>"hostname.example.com")})
+        expect(grok).to include("source"=>{"address"=>"localhost", "ip"=>"127.0.0.1"})
+        expect(grok).to include("network"=>{"protocol"=>"esmtps"})
+      else
+        expect(grok).to include(
+                            "exim_msgid"=>"1UIIN7-0004t9-8R",
+                            "exim_sender_email"=>"root@example.com",
+                            "exim_flags"=>"<=",
+                            "protocol"=>"esmtps",
+                            "exim_header_id"=>"201303201244.r2KCi11V018784@hostname.example.com"
+                            )
+        expect(grok).to include(
+                            "remote_host"=>"127.0.0.1",
+                            "remote_heloname"=>"hostname.example.com",
+                            "remote_hostname"=>"localhost"
+                        )
+      end
+    end
+
+  end
+
+
+  context 'message arrival with quoted hostname and port' do
+
+    let(:message) do
+      '2014-08-10 11:18:35 [28107] 1gsu1C-003dCu-Hb <= aaron@domain.com ' +
+          'H=localhost (10.5.40.204) [127.0.0.1]:39753 I=[127.0.0.1]:25 ' +
+          'P=esmtpa A=dovecot_plain:aaron@domain.com S=4315 M8S=0 id=d2b648f00f1a1b0813c483d552778dc6@domain.com ' +
+          "T=\"what's up?!? ;-)\" from <aaron@domain.com> for aaron+forward@domain.com"
+    end
+
+    it "matches" do
+      if ecs_compatibility?
+        expect(grok).to include("exim"=>{"log"=>hash_including(
+            "message"=>hash_including("id"=>"1gsu1C-003dCu-Hb", "subject"=>"what's up?!? ;-)"),
+        )})
+        expect(grok).to include("destination"=>{"ip"=>"127.0.0.1", "port"=>25})
+        expect(grok).to include("exim"=>{"log"=>hash_including(
+            "sender"=>{"email"=>"aaron@domain.com", 'original'=>'aaron@domain.com'}
+        )})
+        expect(grok).to include("exim"=>{"log"=>hash_including("recipient"=>{"email"=>"aaron+forward@domain.com"})})
+        expect(grok).to include("source"=>{"address"=>"localhost", "ip"=>"127.0.0.1", "port"=>39753})
+        expect(grok).to include("exim"=>{"log" => hash_including("remote_address"=>'10.5.40.204')})
+      else
+        expect(grok).to include(
+                            "exim_msgid"=>"1gsu1C-003dCu-Hb",
+                            "exim_sender_email"=>"aaron@domain.com",
+                            "exim_flags"=>"<=",
+                            "protocol"=>"esmtpa"
+                        )
+        expect(grok).to include(
+                            "remote_host"=>"127.0.0.1", "remote_port"=>"39753",
+                            "remote_heloname"=>"10.5.40.204",
+                            "remote_hostname"=>"localhost"
+                        )
+      end
+    end
+
+  end
+
+  context 'delivery failed' do
+
+    let(:message) do
+      '2020-02-11 17:09:47 1j1Z2g-00Faoy-Uh ** admin@example.net R=virtual_aliases: No such person at this address.'
+    end
+
+    it "does not parse" do # matching not implemented
+      expect(grok['tags']).to include("_grokparsefailure")
+    end
+
+  end
+
+end