logstash-filter-grok 4.0.4 → 4.4.0

data/logstash-filter-grok.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
-
   s.name            = 'logstash-filter-grok'
-  s.version         = '4.0.4'
+  s.version         = '4.4.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses unstructured event data into fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,10 +21,11 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency "logstash-core", ">= 5.6.0"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.0'
 
   s.add_runtime_dependency 'jls-grok', '~> 0.11.3'
   s.add_runtime_dependency 'stud', '~> 0.0.22'
-  s.add_runtime_dependency 'logstash-patterns-core'
+  s.add_runtime_dependency 'logstash-patterns-core', '>= 4.3.0', '< 5'
 
-  s.add_development_dependency 'logstash-devutils', '= 1.3.6'
+  s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/grok_performance_spec.rb

@@ -0,0 +1,144 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+
+begin
+  require "rspec-benchmark"
+rescue LoadError # due testing against LS 5.x
+end
+RSpec.configure do |config|
+  config.include RSpec::Benchmark::Matchers if defined? RSpec::Benchmark::Matchers
+end
+
+require "logstash/filters/grok"
+
+describe LogStash::Filters::Grok do
+
+  subject do
+    described_class.new(config).tap { |filter| filter.register }
+  end
+
+  EVENT_COUNT = 300_000
+
+  describe "base-line performance", :performance => true do
+
+    EXPECTED_MIN_RATE = 30_000 # per second - based on Travis CI (docker) numbers
+
+    let(:config) do
+      { 'match' => { "message" => "%{SYSLOGLINE}" }, 'overwrite' => [ "message" ] }
+    end
+
+    it "matches at least #{EXPECTED_MIN_RATE} events/second" do
+      max_duration = EVENT_COUNT / EXPECTED_MIN_RATE
+      message = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
+      expect do
+        duration = measure do
+          EVENT_COUNT.times { subject.filter(LogStash::Event.new("message" => message)) }
+        end
+        puts "filters/grok parse rate: #{"%02.0f/sec" % (EVENT_COUNT / duration)}, elapsed: #{duration}s"
+      end.to perform_under(max_duration).warmup(1).sample(2).times
+    end
+
+  end
+
+  describe "timeout", :performance => true do
+
+    ACCEPTED_TIMEOUT_DEGRADATION = 100 # in % (compared to timeout-less run)
+    # TODO: with more real-world (pipeline) setup this usually gets bellow 10% on average
+
+    MATCH_PATTERNS = {
+      "message" => [
+        "foo0: %{NUMBER:bar}", "foo1: %{NUMBER:bar}", "foo2: %{NUMBER:bar}", "foo3: %{NUMBER:bar}", "foo4: %{NUMBER:bar}",
+        "foo5: %{NUMBER:bar}", "foo6: %{NUMBER:bar}", "foo7: %{NUMBER:bar}", "foo8: %{NUMBER:bar}", "foo9: %{NUMBER:bar}",
+        "%{SYSLOGLINE}"
+      ]
+    }
+
+    SAMPLE_MESSAGE = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from aaaaaaaa.aaaaaa.net[111.111.11.1]".freeze
+
+    TIMEOUT_MILLIS = 5_000
+
+    let(:config_wout_timeout) do
+      {
+        'match' => MATCH_PATTERNS,
+        'timeout_scope' => "event",
+        'timeout_millis' => 0 # 0 - disabled timeout
+      }
+    end
+
+    let(:config_with_timeout) do
+      {
+        'match' => MATCH_PATTERNS,
+        'timeout_scope' => "event",
+        'timeout_millis' => TIMEOUT_MILLIS
+      }
+    end
+
+    SAMPLE_COUNT = 2
+
+    it "has less than #{ACCEPTED_TIMEOUT_DEGRADATION}% overhead" do
+      filter_wout_timeout = LogStash::Filters::Grok.new(config_wout_timeout).tap(&:register)
+      wout_timeout_duration = do_sample_filter(filter_wout_timeout) # warmup
+      puts "filters/grok(timeout => 0) warmed up in #{wout_timeout_duration}"
+      before_sample!
+      no_timeout_durations = Array.new(SAMPLE_COUNT).map do
+        do_sample_filter(filter_wout_timeout)
+      end
+      puts "filters/grok(timeout => 0) took #{no_timeout_durations}"
+
+      expected_duration = avg(no_timeout_durations)
+      expected_duration += (expected_duration / 100) * ACCEPTED_TIMEOUT_DEGRADATION
+      puts "expected_duration #{expected_duration}"
+
+      filter_with_timeout = LogStash::Filters::Grok.new(config_with_timeout).tap(&:register)
+      with_timeout_duration = do_sample_filter(filter_with_timeout) # warmup
+      puts "filters/grok(timeout_scope => event) warmed up in #{with_timeout_duration}"
+
+      try(3) do
+        before_sample!
+        durations = []
+        begin
+          expect do
+            do_sample_filter(filter_with_timeout).tap { |duration| durations << duration }
+          end.to perform_under(expected_duration).sample(SAMPLE_COUNT).times
+        ensure
+          puts "filters/grok(timeout_scope => event) took #{durations}"
+        end
+      end
+    end
+
+    @private
+
+    def do_sample_filter(filter)
+      sample_event = { "message" => SAMPLE_MESSAGE }
+      measure do
+        for _ in (1..EVENT_COUNT) do # EVENT_COUNT.times without the block cost
+          filter.filter(LogStash::Event.new(sample_event))
+        end
+      end
+    end
+
+  end
+
+  @private
+
+  def measure
+    start = Time.now
+    yield
+    Time.now - start
+  end
+
+  def avg(ary)
+    ary.inject(0) { |m, i| m + i } / ary.size.to_f
+  end
+
+  def before_sample!
+    2.times { JRuby.gc }
+    sleep TIMEOUT_MILLIS / 1000
+  end
+
+  def sleep(seconds)
+    puts "sleeping for #{seconds} seconds (redundant - potential timeout propagation)"
+    Kernel.sleep(seconds)
+  end
+
+end

data/spec/filters/grok_spec.rb

@@ -1,392 +1,367 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "stud/temporary"
+require_relative "../spec_helper"
 
-module LogStash::Environment
-  # running the grok code outside a logstash package means
-  # LOGSTASH_HOME will not be defined, so let's set it here
-  # before requiring the grok filter
-  unless self.const_defined?(:LOGSTASH_HOME)
-    LOGSTASH_HOME = File.expand_path("../../../", __FILE__)
+require "logstash/filters/grok"
+
+describe LogStash::Filters::Grok do
+  subject { described_class.new(config) }
+  let(:config) { {} }
+  let(:event) { LogStash::Event.new(data) }
+  let(:data) { { "message" => message } }
+
+  before(:each) do
+    subject.register
+    subject.filter(event)
   end
 
-  # also :pattern_path method must exist so we define it too
-  unless self.method_defined?(:pattern_path)
-    def pattern_path(path)
-      ::File.join(LOGSTASH_HOME, "patterns", path)
+  def self.sample(message, &block)
+    # mod = RSpec::Core::MemoizedHelpers.module_for(self)
+    # mod.attr_reader :message
+    # # mod.__send__(:define_method, :message) { message }
+    # it("matches: #{message}") { @message = message; block.call }
+    describe message do
+      let(:message) { message }
+      it("groks", &block)
     end
   end
-end
 
-require "logstash/filters/grok"
+  describe "simple syslog line" do
+    let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "overwrite" => [ "message" ] } }
+    let(:message) { 'Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]' }
 
-describe LogStash::Filters::Grok do
+    it "matches pattern" do
+      expect( event.get("tags") ).to be nil
+      expect( event.get("logsource") ).to eql "evita"
+      expect( event.get("timestamp") ).to eql "Mar 16 00:01:25"
+      expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
+      expect( event.get("program") ).to eql "postfix/smtpd"
+      expect( event.get("pid") ).to eql "1713"
+    end
 
-  describe "simple syslog line" do
-    # The logstash config goes here.
-    # At this time, only filters are supported.
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{SYSLOGLINE}" }
-          overwrite => [ "message" ]
-        }
-      }
-    CONFIG
+    context 'in ecs mode' do
+      let(:config) { super.merge('ecs_compatibility' => 'v1') }
 
-    sample "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("logsource") } == "evita"
-      insist { subject.get("timestamp") } == "Mar 16 00:01:25"
-      insist { subject.get("message") } == "connect from camomile.cloud9.net[168.100.1.3]"
-      insist { subject.get("program") } == "postfix/smtpd"
-      insist { subject.get("pid") } == "1713"
+      it "matches pattern" do
+        expect( event.get("host") ).to eql "hostname"=>"evita"
+        expect( event.get("process") ).to eql "name"=>"postfix/smtpd", "pid"=>1713
+        expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
+      end
+    end
+
+    context 'with target' do
+      let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "grok" } }
+
+      it "matches pattern" do
+        expect( event.get("message") ).to eql message
+        expect( event.get("tags") ).to be nil
+        expect( event.get("grok") ).to_not be nil
+        expect( event.get("[grok][timestamp]") ).to eql "Mar 16 00:01:25"
+        expect( event.get("[grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
+        expect( event.get("[grok][pid]") ).to eql "1713"
+      end
+    end
+
+    context 'with [deep] target' do
+      let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "[@metadata][grok]" } }
+
+      it "matches pattern" do
+        expect( event.get("message") ).to eql message
+        expect( event.get("tags") ).to be nil
+        expect( event.get("grok") ).to be nil
+        expect( event.get("[@metadata][grok][logsource]") ).to eql "evita"
+        expect( event.get("[@metadata][grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
+      end
     end
   end
 
   describe "ietf 5424 syslog line" do
-    # The logstash config goes here.
-    # At this time, only filters are supported.
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{SYSLOG5424LINE}" }
-        }
-      }
-    CONFIG
+    let(:config) { { "match" => { "message" => "%{SYSLOG5424LINE}" } } }
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\"bar\"][id2 baz=\"something\"] Hello, syslog." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == "[id1 foo=\"bar\"][id2 baz=\"something\"]"
-      insist { subject.get("syslog5424_msg") } == "Hello, syslog."
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "191"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
+      expect( event.get("syslog5424_host") ).to eql "paxton.local"
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to eql "4123"
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"][id2 baz=\"something\"]"
+      expect( event.get("syslog5424_msg") ).to eql "Hello, syslog."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - [id1 foo=\"bar\"] No process ID." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == nil
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == "[id1 foo=\"bar\"]"
-      insist { subject.get("syslog5424_msg") } == "No process ID."
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "191"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
+      expect( event.get("syslog5424_host") ).to eql "paxton.local"
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to be nil
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"]"
+      expect( event.get("syslog5424_msg") ).to eql "No process ID."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - - No structured data." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "No structured data."
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "191"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
+      expect( event.get("syslog5424_host") ).to eql "paxton.local"
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to eql '4123'
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "No structured data."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - - No PID or SD." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == nil
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "No PID or SD."
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "191"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
+      expect( event.get("syslog5424_host") ).to eql "paxton.local"
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to be nil
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "No PID or SD."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 -  Missing structured data." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Missing structured data."
+      expect( event.get("tags") ).to be nil
+
+      expect( event.get("syslog5424_proc") ).to eql '4123'
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Missing structured data."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug  4123 - - Additional spaces." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Additional spaces."
+      expect( event.get("tags") ).to be nil
+
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to eql '4123'
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Additional spaces."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug  4123 -  Additional spaces and missing SD." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Additional spaces and missing SD."
+      expect( event.get("tags") ).to be nil
+
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to eql '4123'
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Additional spaces and missing SD."
     end
 
     sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 dnsmasq-dhcp 8048 - -  Appname contains a dash" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "30"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2014-04-04T16:44:07+02:00"
-      insist { subject.get("syslog5424_host") } == "osctrl01"
-      insist { subject.get("syslog5424_app") } == "dnsmasq-dhcp"
-      insist { subject.get("syslog5424_proc") } == "8048"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Appname contains a dash"
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "30"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
+      expect( event.get("syslog5424_host") ).to eql "osctrl01"
+      expect( event.get("syslog5424_app") ).to eql "dnsmasq-dhcp"
+      expect( event.get("syslog5424_proc") ).to eql "8048"
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Appname contains a dash"
     end
 
     sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 - 8048 - -  Appname is nil" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "30"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2014-04-04T16:44:07+02:00"
-      insist { subject.get("syslog5424_host") } == "osctrl01"
-      insist { subject.get("syslog5424_app") } == nil
-      insist { subject.get("syslog5424_proc") } == "8048"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Appname is nil"
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "30"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
+      expect( event.get("syslog5424_host") ).to eql "osctrl01"
+      expect( event.get("syslog5424_app") ).to be nil
+      expect( event.get("syslog5424_proc") ).to eql "8048"
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Appname is nil"
     end
   end
 
-  describe "parsing an event with multiple messages (array of strings)", :if => false do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "(?:hello|world) %{NUMBER}" }
-          named_captures_only => false
-        }
-      }
-    CONFIG
+  describe "parsing an event with multiple messages (array of strings)", if: false do
+    let(:config) { { "message" => "(?:hello|world) %{NUMBER}" } }
+    let(:message) { [ "hello 12345", "world 23456" ] }
 
-    sample("message" => [ "hello 12345", "world 23456" ]) do
-      insist { subject.get("NUMBER") } == [ "12345", "23456" ]
+    it "matches them all" do
+      expect( event.get("NUMBER") ).to eql [ "12345", "23456" ]
     end
   end
 
   describe "coercing matched values" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" }
-        }
-      }
-    CONFIG
+    let(:config) { { "match" => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" } } }
+    let(:message) { '400 454.33' }
 
-    sample "400 454.33" do
-      insist { subject.get("foo") } == 400
-      insist { subject.get("foo") }.is_a?(Fixnum)
-      insist { subject.get("bar") } == 454.33
-      insist { subject.get("bar") }.is_a?(Float)
+    it "coerces matched values" do
+      expect( event.get("foo") ).to be_a Integer
+      expect( event.get("foo") ).to eql 400
+      expect( event.get("bar") ).to be_a Float
+      expect( event.get("bar") ).to eql 454.33
     end
   end
 
   describe "in-line pattern definitions" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{FIZZLE=\\d+}" }
-          named_captures_only => false
-        }
-      }
-    CONFIG
+    let(:config) { { "match" => { "message" => "%{FIZZLE=\\d+}" }, "named_captures_only" => false } }
 
     sample "hello 1234" do
-      insist { subject.get("FIZZLE") } == "1234"
+      expect( event.get("FIZZLE") ).to eql '1234'
     end
   end
 
   describe "processing selected fields" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{WORD:word}" }
-          match => { "examplefield" => "%{NUMBER:num}" }
-          break_on_match => false
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{WORD:word}", "examplefield" => "%{NUMBER:num}" },
+        'break_on_match' => false
       }
-    CONFIG
+    }
+    let(:data) { { "message" => "hello world", "examplefield" => "12345" } }
 
-    sample("message" => "hello world", "examplefield" => "12345") do
-      insist { subject.get("examplefield") } == "12345"
-      insist { subject.get("word") } == "hello"
+    it "processes declared matches" do
+      expect( event.get("word") ).to eql 'hello'
+      expect( event.get("examplefield") ).to eql '12345'
     end
   end
 
   describe "adding fields on match" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "matchme %{NUMBER:fancy}" }
-          add_field => [ "new_field", "%{fancy}" ]
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "matchme %{NUMBER:fancy}" },
+        'add_field' => [ "new_field", "%{fancy}" ]
       }
-    CONFIG
+    }
 
     sample "matchme 1234" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("new_field") } == "1234"
+      expect( event.get("tags") ).to be nil
+      expect( event.get("new_field") ).to eql "1234"
     end
 
     sample "this will not be matched" do
-      insist { subject.get("tags") }.include?("_grokparsefailure")
-      reject { subject }.include?("new_field")
+      expect( event.get("tags") ).to include("_grokparsefailure")
+      expect( event ).not_to include 'new_field'
     end
   end
 
   context "empty fields" do
     describe "drop by default" do
-      config <<-CONFIG
-        filter {
-          grok {
-            match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
-          }
+      let(:config) {
+        {
+          'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
         }
-      CONFIG
+      }
 
       sample "1=test" do
-        insist { subject.get("tags") }.nil?
-        insist { subject }.include?("foo1")
+        expect( event.get("tags") ).to be nil
+        expect( event ).to include 'foo1'
 
         # Since 'foo2' was not captured, it must not be present in the event.
-        reject { subject }.include?("foo2")
+        expect( event ).not_to include 'foo2'
       end
     end
 
     describe "keep if keep_empty_captures is true" do
-      config <<-CONFIG
-        filter {
-          grok {
-            match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
-            keep_empty_captures => true
-          }
+      let(:config) {
+        {
+          'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" },
+          'keep_empty_captures' => true
         }
-      CONFIG
+      }
 
       sample "1=test" do
-        insist { subject.get("tags") }.nil?
+        expect( event.get("tags") ).to be nil
         # use .to_hash for this test, for now, because right now
         # the Event.include? returns false for missing fields as well
         # as for fields with nil values.
-        insist { subject.to_hash }.include?("foo2")
-        insist { subject.to_hash }.include?("foo2")
+        expect( event.to_hash ).to include 'foo1'
+        expect( event.to_hash ).to include 'foo2'
       end
     end
   end
 
   describe "when named_captures_only == false" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "Hello %{WORD}. %{WORD:foo}" }
-          named_captures_only => false
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "Hello %{WORD}. %{WORD:foo}" },
+        'named_captures_only' => false
       }
-    CONFIG
+    }
 
     sample "Hello World, yo!" do
-      insist { subject }.include?("WORD")
-      insist { subject.get("WORD") } == "World"
-      insist { subject }.include?("foo")
-      insist { subject.get("foo") } == "yo"
+      expect( event ).to include 'WORD'
+      expect( event.get("WORD") ).to eql "World"
+      expect( event ).to include 'foo'
+      expect( event.get("foo") ).to eql "yo"
     end
   end
 
   describe "using oniguruma named captures (?<name>regex)" do
     context "plain regexp" do
-      config <<-'CONFIG'
-        filter {
-          grok {
-            match => { "message" => "(?<foo>\w+)" }
-          }
+      let(:config) {
+        {
+          'match' => { "message" => "(?<foo>\\w+)" }
         }
-      CONFIG
+      }
+
       sample "hello world" do
-        insist { subject.get("tags") }.nil?
-        insist { subject.get("foo") } == "hello"
+        expect( event.get("tags") ).to be nil
+        expect( event.get("foo") ).to eql "hello"
       end
     end
 
     context "grok patterns" do
-      config <<-'CONFIG'
-        filter {
-          grok {
-            match => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
-          }
+      let(:config) {
+        {
+          'match' => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
         }
-      CONFIG
+      }
 
       sample "fancy 12-12-12 12:12:12" do
-        insist { subject.get("tags") }.nil?
-        insist { subject.get("timestamp") } == "12-12-12 12:12:12"
+        expect( event.get("tags") ).to be nil
+        expect( event.get("timestamp") ).to eql "12-12-12 12:12:12"
       end
     end
   end
 
   describe "grok on integer types" do
-    config <<-'CONFIG'
-      filter {
-        grok {
-          match => { "status" => "^403$" }
-          add_tag => "four_oh_three"
-        }
+    let(:config) {
+      {
+        'match' => { "status" => "^403$" }, 'add_tag' => "four_oh_three"
       }
-    CONFIG
+    }
+    let(:data) { Hash({ "status" => 403 }) }
 
-    sample("status" => 403) do
-      reject { subject.get("tags") }.include?("_grokparsefailure")
-      insist { subject.get("tags") }.include?("four_oh_three")
+    it "parses" do
+      expect( event.get("tags") ).not_to include "_grokparsefailure"
+      expect( event.get("tags") ).to include "four_oh_three"
     end
   end
 
   describe "grok on float types" do
-    config <<-'CONFIG'
-      filter {
-        grok {
-          match => { "version" => "^1.0$" }
-          add_tag => "one_point_oh"
-        }
+    let(:config) {
+      {
+        'match' => { "version" => "^1.0$" }, 'add_tag' => "one_point_oh"
       }
-    CONFIG
+    }
+    let(:data) { Hash({ "version" => 1.0 }) }
 
-    sample("version" => 1.0) do
-      insist { subject.get("tags") }.include?("one_point_oh")
-      insist { subject.get("tags") }.include?("one_point_oh")
+    it "parses" do
+      expect( event.get("tags") ).not_to include "_grokparsefailure"
+      expect( event.get("tags") ).to include "one_point_oh"
     end
   end
 
   describe "grok on %{LOGLEVEL}" do
-    config <<-'CONFIG'
-      filter {
-        grok {
-          match => { "message" => "%{LOGLEVEL:level}: error!" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{LOGLEVEL:level}: error!" }
       }
-    CONFIG
+    }
 
     log_level_names = %w(
       trace Trace TRACE
@@ -402,531 +377,533 @@ describe LogStash::Filters::Grok do
     )
     log_level_names.each do |level_name|
       sample "#{level_name}: error!" do
-        insist { subject.get("level") } == level_name
+        expect( event.get("level") ).to eql level_name
       end
     end
   end
 
   describe "timeout on failure" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => {
-            message => "(.*a){30}"
-          }
-          timeout_millis => 100
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "(.*a){30}" },
+        'timeout_millis' => 100
       }
-    CONFIG
+    }
 
     sample "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" do
-      expect(subject.get("tags")).to include("_groktimeout")
-      expect(subject.get("tags")).not_to include("_grokparsefailure")
+      expect( event.get("tags") ).to include("_groktimeout")
+      expect( event.get("tags") ).not_to include("_grokparsefailure")
+    end
+  end
+
+  describe "no timeout on failure with multiple patterns (when timeout not grouped)" do
+    let(:config) {
+      {
+        'match' => {
+          "message" => [
+            "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
+            "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
+            "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
+            "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
+            "(.*a){20}"
+          ]
+        },
+        'timeout_millis' => 750,
+        'timeout_scope' => 'pattern'
+      }
+    }
+
+    sample( 'b' * 15 + 'c' * 15 + 'd' * 15 + 'e' * 15 + ' ' + 'a' * 20 ) do
+      expect( event.get("tags") ).to be nil
+    end
+  end
+
+  describe "timeout on grouped (multi-pattern) failure" do
+    let(:config) {
+      {
+        'match' => {
+          "message" => [
+            "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
+            "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
+            "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
+            "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
+            "(.*a){20}"
+          ]
+        },
+        'timeout_millis' => 750,
+        'timeout_scope' => 'event'
+      }
+    }
+
+    sample( 'b' * 15 + 'c' * 15 + 'd' * 15 + 'e' * 15 + ' ' + 'a' * 20 ) do
+      expect( event.get("tags") ).to include("_groktimeout")
+      expect( event.get("tags") ).not_to include("_grokparsefailure")
     end
   end
 
   describe "tagging on failure" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "matchme %{NUMBER:fancy}" }
-          tag_on_failure => not_a_match
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "matchme %{NUMBER:fancy}" },
+        'tag_on_failure' => 'not_a_match'
       }
-    CONFIG
+    }
 
     sample "matchme 1234" do
-      insist { subject.get("tags") }.nil?
+      expect( event.get("tags") ).to be nil
     end
 
     sample "this will not be matched" do
-      insist { subject.get("tags") }.include?("not_a_match")
+      expect( event.get("tags") ).to include("not_a_match")
     end
   end
 
   describe "captures named fields even if the whole text matches" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{DATE_EU:stimestamp}" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{DATE_EU:stimestamp}" }
       }
-    CONFIG
+    }
 
     sample "11/01/01" do
-      insist { subject.get("stimestamp") } == "11/01/01"
+      expect( event.get("stimestamp") ).to eql "11/01/01"
     end
   end
 
   describe "allow dashes in capture names" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{WORD:foo-bar}" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{WORD:foo-bar}" }
       }
-    CONFIG
+    }
 
     sample "hello world" do
-      insist { subject.get("foo-bar") } == "hello"
-    end
-  end
-
-  describe "performance test", :performance => true do
-    event_count = 100000
-    min_rate = 2000
-
-    max_duration = event_count / min_rate
-    input = "Nov 24 01:29:01 -0800"
-    config <<-CONFIG
-      input {
-        generator {
-          count => #{event_count}
-          message => "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
-        }
-      }
-      filter {
-        grok {
-          match => { "message" => "%{SYSLOGLINE}" }
-          overwrite => [ "message" ]
-        }
-      }
-      output { null { } }
-    CONFIG
-
-    2.times do
-      start = Time.now
-      agent do
-        duration = (Time.now - start)
-        puts "filters/grok parse rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s"
-        insist { duration } < max_duration
-      end
+      expect( event.get("foo-bar") ).to eql "hello"
     end
   end
 
   describe "single value match with duplicate-named fields in pattern" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{INT:foo}|%{WORD:foo}" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{INT:foo}|%{WORD:foo}" }
       }
-    CONFIG
+    }
 
     sample "hello world" do
-      insist { subject.get("foo") }.is_a?(String)
+      expect( event.get("foo") ).to be_a(String)
     end
 
     sample "123 world" do
-      insist { subject.get("foo") }.is_a?(String)
+      expect( event.get("foo") ).to be_a(String)
     end
   end
 
-  describe "break_on_match default should be true and first match should exit filter" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{INT:foo}"
-                     "somefield" => "%{INT:bar}"}
-        }
-      }
-    CONFIG
 
-    sample("message" => "hello world 123", "somefield" => "testme abc 999") do
-      insist { subject.get("foo") } == "123"
-      insist { subject.get("bar") }.nil?
-    end
-  end
-
-  describe "break_on_match when set to false should try all patterns" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{INT:foo}"
-                     "somefield" => "%{INT:bar}"}
-          break_on_match => false
-        }
+  describe "break_on_match default should be true" do
+    let(:config) {
+      {
+        'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" }
       }
-    CONFIG
+    }
+    let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
 
-    sample("message" => "hello world 123", "somefield" => "testme abc 999") do
-      insist { subject.get("foo") } == "123"
-      insist { subject.get("bar") } == "999"
+    it 'exits filter after first match' do
+      expect( event.get("foo") ).to eql '123'
+      expect( event.get("bar") ).to be nil
     end
   end
 
-  describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
-          break_on_match => false
-        }
+  describe "break_on_match when set to false" do
+    let(:config) {
+      {
+        'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" },
+        'break_on_match' => false
       }
-    CONFIG
-
-    sample "treebranch" do
-      insist { subject.get("name2") } == "branch"
-    end
+    }
+    let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
 
-    sample "bushbeard" do
-      insist { subject.get("name1") } == "bush"
-    end
-
-    sample "treebeard" do
-      insist { subject.get("name1") } == "tree"
-      insist { subject.get("name2") } == "beard"
+    it 'should try all patterns' do
+      expect( event.get("foo") ).to eql '123'
+      expect( event.get("bar") ).to eql '999'
     end
   end
 
-  describe "break_on_match default for array input with single grok pattern" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{INT:foo}"}
-        }
+  context "break_on_match default for array input with single grok pattern" do
+    let(:config) {
+      {
+        'match' => { "message" => "%{INT:foo}" },
+        'break_on_match' => false
       }
-    CONFIG
+    }
 
-    # array input --
-    sample("message" => ["hello world 123", "line 23"]) do
-      insist { subject.get("foo") } == ["123", "23"]
-      insist { subject.get("tags") }.nil?
+    describe 'fully matching input' do
+      let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
+      it 'matches' do
+        expect( event.get("foo") ).to eql ["123", "23"]
+        expect( event.get("tags") ).to be nil
+      end
     end
 
-    # array input, one of them matches
-    sample("message" => ["hello world 123", "abc"]) do
-      insist { subject.get("foo") } == "123"
-      insist { subject.get("tags") }.nil?
+    describe 'partially matching input' do
+      let(:data) { Hash("message" => ["hello world 123", "abc"]) } # array input, one of them matches
+      it 'matches' do
+        expect( event.get("foo") ).to eql "123"
+        expect( event.get("tags") ).to be nil
+      end
     end
   end
 
   describe "break_on_match = true (default) for array input with multiple grok pattern" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
       }
-    CONFIG
-
-    # array input --
-    sample("message" => ["hello world 123", "line 23"]) do
-      insist { subject.get("foo") } == ["123", "23"]
-      insist { subject.get("bar") }.nil?
-      insist { subject.get("tags") }.nil?
+    }
+
+    describe 'matching input' do
+      let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
+      it 'matches' do
+        expect( event.get("foo") ).to eql ["123", "23"]
+        expect( event.get("bar") ).to be nil
+        expect( event.get("tags") ).to be nil
+      end
     end
 
-    # array input, one of them matches
-    sample("message" => ["hello world", "line 23"]) do
-      insist { subject.get("bar") } == "hello"
-      insist { subject.get("foo") } == "23"
-      insist { subject.get("tags") }.nil?
+    describe 'partially matching input' do
+      let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
+      it 'matches' do
+        expect( event.get("bar") ).to eql 'hello'
+        expect( event.get("foo") ).to eql "23"
+        expect( event.get("tags") ).to be nil
+      end
     end
   end
 
   describe "break_on_match = false for array input with multiple grok pattern" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
-          break_on_match => false
-        }
+    let(:config) {
+      {
+        'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] },
+        'break_on_match' => false
       }
-    CONFIG
-
-    # array input --
-    sample("message" => ["hello world 123", "line 23"]) do
-      insist { subject.get("foo") } == ["123", "23"]
-      insist { subject.get("bar") } == ["hello", "line"]
-      insist { subject.get("tags") }.nil?
+    }
+
+    describe 'fully matching input' do
+      let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
+      it 'matches' do
+        expect( event.get("foo") ).to eql ["123", "23"]
+        expect( event.get("bar") ).to eql ["hello", "line"]
+        expect( event.get("tags") ).to be nil
+      end
     end
 
-    # array input, one of them matches
-    sample("message" => ["hello world", "line 23"]) do
-      insist { subject.get("bar") } == ["hello", "line"]
-      insist { subject.get("foo") } == "23"
-      insist { subject.get("tags") }.nil?
+    describe 'partially matching input' do
+      let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
+      it 'matches' do
+        expect( event.get("bar") ).to eql ["hello", "line"]
+        expect( event.get("foo") ).to eql "23"
+        expect( event.get("tags") ).to be nil
+      end
     end
   end
 
   describe  "grok with unicode" do
-    config <<-CONFIG
-      filter {
-        grok {
-          #match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
-          match => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
-        }
+    let(:config) {
+      {
+        #'match' => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+        'match' => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
       }
-    CONFIG
+    }
 
     sample "<22>Jan  4 07:50:46 mailmaster postfix/policy-spf[9454]: : SPF permerror (Junk encountered in record 'v=spf1 mx a:mail.domain.no ip4:192.168.0.4 �all'): Envelope-from: email@domain.no" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog_pri") } == "22"
-      insist { subject.get("syslog_program") } == "postfix/policy-spf"
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog_pri") ).to eql "22"
+      expect( event.get("syslog_program") ).to eql "postfix/policy-spf"
     end
   end
 
-  describe "patterns in the 'patterns/' dir override core patterns" do
-
-    let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
-    let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
+  describe  "grok with nil coerced value" do
+    let(:config) {
+      {
+        'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
+      }
+    }
 
-    before do
-      FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
-      @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
-      @file.write('WORD \b[2-5]\b')
-      @file.close
+    sample "test 28.4ms" do
+      expect( event.get("duration") ).to eql 28.4
+      expect( event.get("tags") ).to be nil
     end
 
-    let(:config) do
-      'filter { grok { match => { "message" => "%{WORD:word}" } } }'
+    sample "test N/A" do
+      expect( event.to_hash ).not_to include("duration")
+      expect( event.get("tags") ).to be nil
     end
 
-    sample("message" => 'hello') do
-      insist { subject.get("tags") } == ["_grokparsefailure"]
+    sample "test abc" do
+      expect( event.get("duration") ).to be nil
+      expect( event.get("tags") ).to eql ["_grokparsefailure"]
     end
+  end
+
+  describe  "grok with nil coerced value and keep_empty_captures" do
+    let(:config) {
+      {
+        'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" },
+        'keep_empty_captures' => true
+      }
+    }
 
-    after do
-      File.unlink @file
-      FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
+    sample "test N/A" do
+      expect( event.to_hash ).to include("duration")
+      expect( event.get("tags") ).to be nil
     end
   end
 
-  describe "patterns in custom dir override those in 'patterns/' dir" do
-
-    let(:tmpdir) { Stud::Temporary.directory }
-    let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
-    let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
+  describe  "grok with no coercion" do
+    let(:config) {
+      {
+        'match' => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" },
+      }
+    }
 
-    before do
-      FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
-      @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
-      @file1.write('WORD \b[2-5]\b')
-      @file1.close
-      @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
-      @file2.write('WORD \b[0-1]\b')
-      @file2.close
+    sample "test 28.4ms" do
+      expect( event.get("duration") ).to eql '28.4'
+      expect( event.get("tags") ).to be nil
     end
 
-    let(:config) do
-      "filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
+    sample "test N/A" do
+      expect( event.get("duration") ).to be nil
+      expect( event.get("tags") ).to be nil
     end
+  end
 
-    sample("message" => '0') do
-      insist { subject.get("tags") } == nil
-    end
+  describe "opening/closing" do
+    let(:config) { { "match" => {"message" => "A"} } }
+    let(:message) { 'AAA' }
 
-    after do
-      File.unlink @file1
-      File.unlink @file2
-      FileUtils.remove_entry tmpdir
-      FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
+    it "should close cleanly" do
+      expect { subject.do_close }.not_to raise_error
     end
   end
 
-  describe "patterns with file glob" do
+  describe "after grok when the event is JSON serialised the field values are unchanged" do
+    let(:config) {
+      {
+        'match' => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"],
+        'remove_field' => ["message","severity"],
+        'add_tag' => ["ssh_failure"]
+      }
+    }
 
-    let(:tmpdir) { Stud::Temporary.directory }
+    sample('{"facility":"auth","message":"Failed password for testuser from 1.1.1.1 port 22"}') do
+      expect( event.get("username") ).to eql "testuser"
+      expect( event.get("port") ).to eql "22"
+      expect( event.get("src_ip") ).to eql "1.1.1.1"
+      expect( LogStash::Json.dump(event.get('username')) ).to eql "\"testuser\""
 
-    before do
-      @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
-      @file3.write('WORD \b[0-1]\b')
-      @file3.close
-      @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
-      @file4.write('WORD \b[2-5]\b')
-      @file4.close
-    end
-
-    let(:config) do
-      "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
+      expect( event.to_json ).to match %r|"src_ip":"1.1.1.1"|
+      expect( event.to_json ).to match %r|"@timestamp":"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ"|
+      expect( event.to_json ).to match %r|"port":"22"|
+      expect( event.to_json ).to match %r|"@version":"1"|
+      expect( event.to_json ).to match %r|"username"|i
+      expect( event.to_json ).to match %r|"testuser"|
+      expect( event.to_json ).to match %r|"tags":\["ssh_failure"\]|
     end
+  end
 
-    sample("message" => '0') do
-      insist { subject.get("tags") } == nil
-    end
+  describe  "grok with inline pattern definition successfully extracts fields" do
+    let(:config) {
+      {
+        'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" },
+        'pattern_definitions' => {
+          "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
+          "MY_PATTERN" => "%{YEAR}"
+        }
+      }
+    }
 
-    after do
-      File.unlink @file3
-      File.unlink @file4
-      FileUtils.remove_entry tmpdir
+    sample "Mon Dec 26 16:22:08 2016 error 2020" do
+      expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
+      expect( event.get("level") ).to eql "error"
+      expect( event.get("hindsight") ).to eql "2020"
     end
   end
 
-  describe "patterns with file glob on directory that contains subdirectories" do
-
-    let(:tmpdir) { Stud::Temporary.directory }
+  describe  "grok with inline pattern definition overwrites existing pattern definition" do
+    let(:config) {
+      {
+        'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" },
+        # loglevel was previously ([Aa]lert|ALERT|[Tt]...
+        'pattern_definitions' => {
+          "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
+          "LOGLEVEL" => "%{NUMBER}"
+        }
+      }
+    }
 
-    before do
-      @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
-      @file3.write('WORD \b[0-1]\b')
-      @file3.close
-      Dir.mkdir(File.join(tmpdir, "subdir"))
+    sample "Mon Dec 26 16:22:08 2016 9999" do
+      expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
+      expect( event.get("level") ).to eql "9999"
     end
+  end
 
+  context 'when timeouts are explicitly disabled' do
     let(:config) do
-      "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
+      {
+        "timeout_millis" => 0
+      }
     end
 
-    sample("message" => '0') do
-      insist { subject.get("tags") } == nil
-    end
+    context 'when given a pathological input', slow: true do
+      let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
+      let(:config) { super().merge("match" => { "message" => "(.*a){30}" }) }
 
-    after do
-      File.unlink @file3
-      FileUtils.remove_entry tmpdir
+      it 'blocks for at least 3 seconds' do
+        blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
+        expect do
+          Timeout.timeout(3, blocking_exception_class) do
+            subject.filter(event)
+          end
+        end.to raise_exception(blocking_exception_class)
+      end
     end
   end
+end
 
-  describe  "grok with nil coerced value" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
-        }
-      }
-    CONFIG
+describe LogStash::Filters::Grok do
+  describe "(LEGACY)" do
+    describe "patterns in the 'patterns/' dir override core patterns" do
 
-    sample "test 28.4ms" do
-      insist { subject.get("duration") } == 28.4
-      insist { subject.get("tags") }.nil?
-    end
+      let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
+      let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
 
-    sample "test N/A" do
-      insist { insist { subject.to_hash }.include?("duration") }.fails
-      insist { subject.get("tags") }.nil?
-    end
+      before do
+        FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
+        @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
+        @file.write('WORD \b[2-5]\b')
+        @file.close
+      end
 
-    sample "test abc" do
-      insist { subject.get("duration") }.nil?
-      insist { subject.get("tags") } == ["_grokparsefailure"]
-    end
-  end
+      let(:config) do
+        'filter { grok { match => { "message" => "%{WORD:word}" } } }'
+      end
 
-  describe  "grok with nil coerced value and keep_empty_captures" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
-          keep_empty_captures => true
-        }
-      }
-    CONFIG
+      sample("message" => 'hello') do
+        expect(subject.get("tags")).to eql ["_grokparsefailure"]
+      end
 
-    sample "test N/A" do
-      insist { subject.to_hash }.include?("duration")
-      insist { subject.get("tags") }.nil?
+      after do
+        File.unlink @file
+        FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
+      end
     end
 
-  end
+    describe "patterns in custom dir override those in 'patterns/' dir" do
 
-  describe  "grok with no coercion" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" }
-        }
-      }
-    CONFIG
+      let(:tmpdir) { Stud::Temporary.directory }
+      let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
+      let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
 
-    sample "test 28.4ms" do
-      insist { subject.get("duration") } == "28.4"
-      insist { subject.get("tags") }.nil?
-    end
+      before do
+        FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
+        @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
+        @file1.write('WORD \b[2-5]\b')
+        @file1.close
+        @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
+        @file2.write('WORD \b[0-1]\b')
+        @file2.close
+      end
 
-    sample "test N/A" do
-      insist { subject.get("duration") }.nil?
-      insist { subject.get("tags") }.nil?
-    end
-  end
+      let(:config) do
+        "filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
+      end
 
-  describe "opening/closing" do
-    let(:config) { {"match" => {"message" => "A"}} }
-    subject(:plugin) do
-      ::LogStash::Filters::Grok.new(config)
-    end
+      sample("message" => '0') do
+        expect(subject.get("tags")).to be nil
+      end
 
-    before do
-      plugin.register
+      after do
+        File.unlink @file1
+        File.unlink @file2
+        FileUtils.remove_entry tmpdir
+        FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
+      end
     end
 
-    it "should start the timeout enforcer" do
-      expect(plugin.timeout_enforcer.running).to be true
-    end
+    describe "patterns with file glob" do
 
-    context "with the timeout enforcer disabled" do
-      let(:config) { super.merge("timeout_millis" => 0) }
+      let(:tmpdir) { Stud::Temporary.directory }
 
-      it "should not start the timeout enforcer" do
-        expect(plugin.timeout_enforcer.running).to be false
+      before do
+        @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
+        @file3.write('WORD \b[0-1]\b')
+        @file3.close
+        @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
+        @file4.write('WORD \b[2-5]\b')
+        @file4.close
       end
-    end
 
-    it "should close cleanly" do
-      expect { plugin.do_close }.not_to raise_error
-    end
+      let(:config) do
+        "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
+      end
+
+      sample("message" => '0') do
+        expect(subject.get("tags")).to be nil
+      end
 
-    it "should stop the timeout enforcer" do
-      plugin.do_close
-      expect(plugin.timeout_enforcer.running).to be false
+      after do
+        File.unlink @file3
+        File.unlink @file4
+        FileUtils.remove_entry tmpdir
+      end
     end
-  end
 
-  describe "after grok when the event is JSON serialised the field values are unchanged" do
-    config <<-CONFIG
-      filter {grok {match => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"] remove_field => ["message","severity"] add_tag => ["ssh_failure"]}}
-    CONFIG
+    describe "patterns with file glob on directory that contains subdirectories" do
 
-    sample('{"facility":"auth","message":"Failed password for testuser from 1.1.1.1 port 22"}') do
-      insist { subject.get("username") } == "testuser"
-      insist { subject.get("port") } == "22"
-      insist { subject.get("src_ip") } == "1.1.1.1"
-      insist { LogStash::Json.dump(subject.get('username')) } == "\"testuser\""
+      let(:tmpdir) { Stud::Temporary.directory }
 
-      insist { subject.to_json } =~ %r|"src_ip":"1.1.1.1"|
-      insist { subject.to_json } =~ %r|"@timestamp":"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ"|
-      insist { subject.to_json } =~ %r|"port":"22"|
-      insist { subject.to_json } =~ %r|"@version":"1"|
-      insist { subject.to_json } =~ %r|"username"|i
-      insist { subject.to_json } =~ %r|"testuser"|
-      insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]|
-    end
-  end
+      before do
+        @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
+        @file3.write('WORD \b[0-1]\b')
+        @file3.close
+        Dir.mkdir(File.join(tmpdir, "subdir"))
+      end
 
-  describe  "grok with inline pattern definition successfully extracts fields" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" }
-          pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
-           "MY_PATTERN" => "%{YEAR}"}
-        }
-      }
-    CONFIG
+      let(:config) do
+        "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
+      end
 
-    sample "Mon Dec 26 16:22:08 2016 error 2020" do
-      insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
-      insist { subject.get("level") } == "error"
-      insist { subject.get("hindsight") } == "2020"
+      sample("message" => '0') do
+        expect(subject.get("tags")).to be nil
+      end
+
+      after do
+        File.unlink @file3
+        FileUtils.remove_entry tmpdir
+      end
     end
-  end
 
-  describe  "grok with inline pattern definition overwrites existing pattern definition" do
-    config <<-CONFIG
+    describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
+      config <<-CONFIG
       filter {
         grok {
-          match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" }
-          # loglevel was previously ([Aa]lert|ALERT|[Tt]...
-          pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
-           "LOGLEVEL" => "%{NUMBER}"}
+          match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
+          break_on_match => false
         }
       }
-    CONFIG
+      CONFIG
 
-    sample "Mon Dec 26 16:22:08 2016 9999" do
-      insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
-      insist { subject.get("level") } == "9999"
+      sample "treebranch" do
+        expect(subject.get("name2")).to eql "branch"
+      end
+
+      sample "bushbeard" do
+        expect(subject.get("name1")).to eql "bush"
+      end
+
+      sample "treebeard" do
+        expect(subject.get("name1")).to eql "tree"
+        expect(subject.get("name2")).to eql "beard"
+      end
     end
   end
-
 end