logstash-filter-grok 4.2.0 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a139ac82c3147d778c2f4510cf6ab077c6c4e73adeae87a0b4058623c4a0619
-  data.tar.gz: 02300cea3c6a17e10947cf63a7dae596c1ef1fc1c5114f3834ac784536a2941a
+  metadata.gz: a9d8be42f7b846a0684d25d18ec624a81574b52e394df863afa004d458ecd915
+  data.tar.gz: 605d0cea73b747d9900bebfc2a99f59d2950300af7ef52ec5b9cb3f556a7d5fd
 SHA512:
-  metadata.gz: e822b6f1aca31141d7c3d553167a27ea1c11ba41e39dd9553fdbecfa3472780ccca75054b2247985261634efb3263a62fb7ca87f5762e418f744ef45bf994889
-  data.tar.gz: 990df0541f9a5cdb09d6fea2060c1c68e901e7cdf1cdb0fd45cfcf32dfa0710da3eeddd7fb3400441489ccc1741f0380daf5934b15e0fbba9ccc2d4a6b370c37
+  metadata.gz: '096791bcc8691300c8e59aa8c6741d499c515b6fc70b3f7e4deea0c23440bce1875d98d7aaad0d6f8d8ab12dc526051e46e2a61b590422d646d7939221ec3fc7'
+  data.tar.gz: efe3dcc313d22e67986828351b365b9383c5c91d0a4caf8da78bd8c34126e7088d81ce9c672a6dc0758a9c3c4d1e4a2cae53e2df2c79bcd510848b69ecac9945

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.3.0
+ -  Added: added target support [#156](https://github.com/logstash-plugins/logstash-filter-grok/pull/156)
+
 ## 4.2.0
  -  Added: support for timeout_scope [#153](https://github.com/logstash-plugins/logstash-filter-grok/pull/153)
 

data/Gemfile

@@ -9,3 +9,9 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+group :test do
+  gem 'rspec-benchmark', :require => false if RUBY_VERSION >= '2.3'
+  gem 'logstash-input-generator', :require => false
+  gem 'logstash-output-null', :require => false
+end

data/docs/index.asciidoc

@@ -345,6 +345,14 @@ successful match
 
 Tag to apply if a grok regexp times out.
 
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting
+
+Define target namespace for placing matches.
+
 [id="plugins-{type}s-{plugin}-timeout_millis"]
 ===== `timeout_millis` 
 

data/lib/logstash/filters/grok.rb

@@ -204,6 +204,10 @@
     # If `true`, keep empty captures as event fields.
     config :keep_empty_captures, :validate => :boolean, :default => false
 
+    # Define the target field for placing the matched captures.
+    # If this setting is omitted, data gets stored at the root (top level) of the event.
+    config :target, :validate => :string
+
     # Append values to the `tags` field when there has been no
     # successful match
     config :tag_on_failure, :validate => :array, :default => ["_grokparsefailure"]
@@ -288,6 +292,8 @@
       @match_counter = metric.counter(:matches)
       @failure_counter = metric.counter(:failures)
 
+      @target = "[#{@target.strip}]" if @target && @target !~ /\[.*?\]/
+
       @timeout = @timeout_millis > 0.0 ? RubyTimeout.new(@timeout_millis) : NoopTimeout::INSTANCE
       @matcher = ( @timeout_scope.eql?('event') ? EventTimeoutMatcher : PatternTimeoutMatcher ).new(self)
     end # def register
@@ -398,22 +404,26 @@
     def handle(field, value, event)
       return if (value.nil? || (value.is_a?(String) && value.empty?)) unless @keep_empty_captures
 
+      target_field = @target ? "#{@target}[#{field}]" : field
+
       if @overwrite.include?(field)
-        event.set(field, value)
+        event.set(target_field, value)
       else
-        v = event.get(field)
+        v = event.get(target_field)
         if v.nil?
-          event.set(field, value)
+          event.set(target_field, value)
         elsif v.is_a?(Array)
           # do not replace the code below with:
           #   event[field] << value
           # this assumes implementation specific feature of returning a mutable object
           # from a field ref which should not be assumed and will change in the future.
           v << value
-          event.set(field, v)
+          event.set(target_field, v)
         elsif v.is_a?(String)
           # Promote to array since we aren't overwriting.
-          event.set(field, [v, value])
+          event.set(target_field, [v, value])
+        else
+          @logger.debug("Not adding matched value - found existing (#{v.class})", :field => target_field, :value => value)
         end
       end
     end

data/logstash-filter-grok.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-grok'
-  s.version         = '4.2.0'
+  s.version         = '4.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses unstructured event data into fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/grok_performance_spec.rb

@@ -0,0 +1,150 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+
+begin
+  require "rspec-benchmark"
+rescue LoadError # due testing against LS 5.x
+end
+RSpec.configure do |config|
+  config.include RSpec::Benchmark::Matchers if defined? RSpec::Benchmark::Matchers
+end
+
+require "logstash/filters/grok"
+
+describe LogStash::Filters::Grok do
+
+  subject do
+    described_class.new(config).tap { |filter| filter.register }
+  end
+
+  EVENT_COUNT = 300_000
+
+  describe "base-line performance", :performance => true do
+
+    EXPECTED_MIN_RATE = 15_000 # per second
+    # NOTE: based on Travis CI (docker) numbers :
+    # logstash_1_d010d1d29244 | LogStash::Filters::Grok
+    # logstash_1_d010d1d29244 |   base-line performance
+    # logstash_1_d010d1d29244 | filters/grok parse rate: 14464/sec, elapsed: 20.740866999999998s
+    # logstash_1_d010d1d29244 | filters/grok parse rate: 29957/sec, elapsed: 10.014199s
+    # logstash_1_d010d1d29244 | filters/grok parse rate: 32932/sec, elapsed: 9.109601999999999s
+
+    let(:config) do
+      { 'match' => { "message" => "%{SYSLOGLINE}" }, 'overwrite' => [ "message" ] }
+    end
+
+    it "matches at least #{EXPECTED_MIN_RATE} events/second" do
+      max_duration = EVENT_COUNT / EXPECTED_MIN_RATE
+      message = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
+      expect do
+        duration = measure do
+          EVENT_COUNT.times { subject.filter(LogStash::Event.new("message" => message)) }
+        end
+        puts "filters/grok parse rate: #{"%02.0f/sec" % (EVENT_COUNT / duration)}, elapsed: #{duration}s"
+      end.to perform_under(max_duration).warmup(1).sample(2).times
+    end
+
+  end
+
+  describe "timeout", :performance => true do
+
+    ACCEPTED_TIMEOUT_DEGRADATION = 100 # in % (compared to timeout-less run)
+    # TODO: with more real-world (pipeline) setup this usually gets bellow 10% on average
+
+    MATCH_PATTERNS = {
+      "message" => [
+        "foo0: %{NUMBER:bar}", "foo1: %{NUMBER:bar}", "foo2: %{NUMBER:bar}", "foo3: %{NUMBER:bar}", "foo4: %{NUMBER:bar}",
+        "foo5: %{NUMBER:bar}", "foo6: %{NUMBER:bar}", "foo7: %{NUMBER:bar}", "foo8: %{NUMBER:bar}", "foo9: %{NUMBER:bar}",
+        "%{SYSLOGLINE}"
+      ]
+    }
+
+    SAMPLE_MESSAGE = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from aaaaaaaa.aaaaaa.net[111.111.11.1]".freeze
+
+    TIMEOUT_MILLIS = 5_000
+
+    let(:config_wout_timeout) do
+      {
+        'match' => MATCH_PATTERNS,
+        'timeout_scope' => "event",
+        'timeout_millis' => 0 # 0 - disabled timeout
+      }
+    end
+
+    let(:config_with_timeout) do
+      {
+        'match' => MATCH_PATTERNS,
+        'timeout_scope' => "event",
+        'timeout_millis' => TIMEOUT_MILLIS
+      }
+    end
+
+    SAMPLE_COUNT = 2
+
+    it "has less than #{ACCEPTED_TIMEOUT_DEGRADATION}% overhead" do
+      filter_wout_timeout = LogStash::Filters::Grok.new(config_wout_timeout).tap(&:register)
+      wout_timeout_duration = do_sample_filter(filter_wout_timeout) # warmup
+      puts "filters/grok(timeout => 0) warmed up in #{wout_timeout_duration}"
+      before_sample!
+      no_timeout_durations = Array.new(SAMPLE_COUNT).map do
+        duration = do_sample_filter(filter_wout_timeout)
+        puts "filters/grok(timeout => 0) took #{duration}"
+        duration
+      end
+
+      expected_duration = avg(no_timeout_durations)
+      expected_duration += (expected_duration / 100) * ACCEPTED_TIMEOUT_DEGRADATION
+      puts "expected_duration #{expected_duration}"
+
+      filter_with_timeout = LogStash::Filters::Grok.new(config_with_timeout).tap(&:register)
+      with_timeout_duration = do_sample_filter(filter_with_timeout) # warmup
+      puts "filters/grok(timeout_scope => event) warmed up in #{with_timeout_duration}"
+
+      before_sample!
+      expect do
+        duration = do_sample_filter(filter_with_timeout)
+        puts "filters/grok(timeout_scope => event) took #{duration}"
+        duration
+      end.to perform_under(expected_duration).sample(SAMPLE_COUNT).times
+    end
+
+    @private
+
+    def do_sample_filter(filter)
+      sample_event = { "message" => SAMPLE_MESSAGE }
+      measure do
+        for _ in (1..EVENT_COUNT) do # EVENT_COUNT.times without the block cost
+          filter.filter(LogStash::Event.new(sample_event))
+        end
+      end
+    end
+
+    def sample_event(hash)
+      LogStash::Event.new("message" => SAMPLE_MESSAGE)
+    end
+
+  end
+
+  @private
+
+  def measure
+    start = Time.now
+    yield
+    Time.now - start
+  end
+
+  def avg(ary)
+    ary.inject(0) { |m, i| m + i } / ary.size.to_f
+  end
+
+  def before_sample!
+    2.times { JRuby.gc }
+    sleep TIMEOUT_MILLIS / 1000
+  end
+
+  def sleep(seconds)
+    puts "sleeping for #{seconds} seconds (redundant - potential timeout propagation)"
+    Kernel.sleep(seconds)
+  end
+
+end

data/spec/filters/grok_spec.rb

@@ -1,392 +1,357 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "stud/temporary"
+require_relative "../spec_helper"
 
-module LogStash::Environment
-  # running the grok code outside a logstash package means
-  # LOGSTASH_HOME will not be defined, so let's set it here
-  # before requiring the grok filter
-  unless self.const_defined?(:LOGSTASH_HOME)
-    LOGSTASH_HOME = File.expand_path("../../../", __FILE__)
+require "logstash/filters/grok"
+
+describe LogStash::Filters::Grok do
+  subject { described_class.new(config) }
+  let(:config) { {} }
+  let(:event) { LogStash::Event.new(data) }
+  let(:data) { { "message" => message } }
+
+  before(:each) do
+    subject.register
+    subject.filter(event)
   end
 
-  # also :pattern_path method must exist so we define it too
-  unless self.method_defined?(:pattern_path)
-    def pattern_path(path)
-      ::File.join(LOGSTASH_HOME, "patterns", path)
+  def self.sample(message, &block)
+    # mod = RSpec::Core::MemoizedHelpers.module_for(self)
+    # mod.attr_reader :message
+    # # mod.__send__(:define_method, :message) { message }
+    # it("matches: #{message}") { @message = message; block.call }
+    describe message do
+      let(:message) { message }
+      it("groks", &block)
     end
   end
-end
-
-require "logstash/filters/grok"
-
-describe LogStash::Filters::Grok do
 
   describe "simple syslog line" do
-    # The logstash config goes here.
-    # At this time, only filters are supported.
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{SYSLOGLINE}" }
-          overwrite => [ "message" ]
-        }
-      }
-    CONFIG
+    let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "overwrite" => [ "message" ] } }
+    let(:message) { 'Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]' }
+
+    it "matches pattern" do
+      expect( event.get("tags") ).to be nil
+      expect( event.get("logsource") ).to eql "evita"
+      expect( event.get("timestamp") ).to eql "Mar 16 00:01:25"
+      expect( event.get("message") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
+      expect( event.get("program") ).to eql "postfix/smtpd"
+      expect( event.get("pid") ).to eql "1713"
+    end
+
+    context 'with target' do
+      let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "grok" } }
+
+      it "matches pattern" do
+        expect( event.get("message") ).to eql message
+        expect( event.get("tags") ).to be nil
+        expect( event.get("grok") ).to_not be nil
+        expect( event.get("[grok][timestamp]") ).to eql "Mar 16 00:01:25"
+        expect( event.get("[grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
+        expect( event.get("[grok][pid]") ).to eql "1713"
+      end
+    end
+
+    context 'with [deep] target' do
+      let(:config) { { "match" => { "message" => "%{SYSLOGLINE}" }, "target" => "[@metadata][grok]" } }
 
-    sample "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("logsource") } == "evita"
-      insist { subject.get("timestamp") } == "Mar 16 00:01:25"
-      insist { subject.get("message") } == "connect from camomile.cloud9.net[168.100.1.3]"
-      insist { subject.get("program") } == "postfix/smtpd"
-      insist { subject.get("pid") } == "1713"
+      it "matches pattern" do
+        expect( event.get("message") ).to eql message
+        expect( event.get("tags") ).to be nil
+        expect( event.get("grok") ).to be nil
+        expect( event.get("[@metadata][grok][logsource]") ).to eql "evita"
+        expect( event.get("[@metadata][grok][message]") ).to eql "connect from camomile.cloud9.net[168.100.1.3]"
+      end
     end
   end
 
   describe "ietf 5424 syslog line" do
-    # The logstash config goes here.
-    # At this time, only filters are supported.
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{SYSLOG5424LINE}" }
-        }
-      }
-    CONFIG
+    let(:config) { { "match" => { "message" => "%{SYSLOG5424LINE}" } } }
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\"bar\"][id2 baz=\"something\"] Hello, syslog." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == "[id1 foo=\"bar\"][id2 baz=\"something\"]"
-      insist { subject.get("syslog5424_msg") } == "Hello, syslog."
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "191"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
+      expect( event.get("syslog5424_host") ).to eql "paxton.local"
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to eql "4123"
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"][id2 baz=\"something\"]"
+      expect( event.get("syslog5424_msg") ).to eql "Hello, syslog."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - [id1 foo=\"bar\"] No process ID." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == nil
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == "[id1 foo=\"bar\"]"
-      insist { subject.get("syslog5424_msg") } == "No process ID."
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "191"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
+      expect( event.get("syslog5424_host") ).to eql "paxton.local"
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to be nil
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to eql "[id1 foo=\"bar\"]"
+      expect( event.get("syslog5424_msg") ).to eql "No process ID."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - - No structured data." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "No structured data."
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "191"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
+      expect( event.get("syslog5424_host") ).to eql "paxton.local"
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to eql '4123'
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "No structured data."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug - - - No PID or SD." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == nil
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "No PID or SD."
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "191"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2009-06-30T18:30:00+02:00"
+      expect( event.get("syslog5424_host") ).to eql "paxton.local"
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to be nil
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "No PID or SD."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 -  Missing structured data." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Missing structured data."
+      expect( event.get("tags") ).to be nil
+
+      expect( event.get("syslog5424_proc") ).to eql '4123'
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Missing structured data."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug  4123 - - Additional spaces." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Additional spaces."
+      expect( event.get("tags") ).to be nil
+
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to eql '4123'
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Additional spaces."
     end
 
     sample "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug  4123 -  Additional spaces and missing SD." do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "191"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2009-06-30T18:30:00+02:00"
-      insist { subject.get("syslog5424_host") } == "paxton.local"
-      insist { subject.get("syslog5424_app") } == "grokdebug"
-      insist { subject.get("syslog5424_proc") } == "4123"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Additional spaces and missing SD."
+      expect( event.get("tags") ).to be nil
+
+      expect( event.get("syslog5424_app") ).to eql "grokdebug"
+      expect( event.get("syslog5424_proc") ).to eql '4123'
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Additional spaces and missing SD."
     end
 
     sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 dnsmasq-dhcp 8048 - -  Appname contains a dash" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "30"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2014-04-04T16:44:07+02:00"
-      insist { subject.get("syslog5424_host") } == "osctrl01"
-      insist { subject.get("syslog5424_app") } == "dnsmasq-dhcp"
-      insist { subject.get("syslog5424_proc") } == "8048"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Appname contains a dash"
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "30"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
+      expect( event.get("syslog5424_host") ).to eql "osctrl01"
+      expect( event.get("syslog5424_app") ).to eql "dnsmasq-dhcp"
+      expect( event.get("syslog5424_proc") ).to eql "8048"
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Appname contains a dash"
     end
 
     sample "<30>1 2014-04-04T16:44:07+02:00 osctrl01 - 8048 - -  Appname is nil" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog5424_pri") } == "30"
-      insist { subject.get("syslog5424_ver") } == "1"
-      insist { subject.get("syslog5424_ts") } == "2014-04-04T16:44:07+02:00"
-      insist { subject.get("syslog5424_host") } == "osctrl01"
-      insist { subject.get("syslog5424_app") } == nil
-      insist { subject.get("syslog5424_proc") } == "8048"
-      insist { subject.get("syslog5424_msgid") } == nil
-      insist { subject.get("syslog5424_sd") } == nil
-      insist { subject.get("syslog5424_msg") } == "Appname is nil"
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog5424_pri") ).to eql "30"
+      expect( event.get("syslog5424_ver") ).to eql "1"
+      expect( event.get("syslog5424_ts") ).to eql "2014-04-04T16:44:07+02:00"
+      expect( event.get("syslog5424_host") ).to eql "osctrl01"
+      expect( event.get("syslog5424_app") ).to be nil
+      expect( event.get("syslog5424_proc") ).to eql "8048"
+      expect( event.get("syslog5424_msgid") ).to be nil
+      expect( event.get("syslog5424_sd") ).to be nil
+      expect( event.get("syslog5424_msg") ).to eql "Appname is nil"
     end
   end
 
-  describe "parsing an event with multiple messages (array of strings)", :if => false do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "(?:hello|world) %{NUMBER}" }
-          named_captures_only => false
-        }
-      }
-    CONFIG
+  describe "parsing an event with multiple messages (array of strings)", if: false do
+    let(:config) { { "message" => "(?:hello|world) %{NUMBER}" } }
+    let(:message) { [ "hello 12345", "world 23456" ] }
 
-    sample("message" => [ "hello 12345", "world 23456" ]) do
-      insist { subject.get("NUMBER") } == [ "12345", "23456" ]
+    it "matches them all" do
+      expect( event.get("NUMBER") ).to eql [ "12345", "23456" ]
     end
   end
 
   describe "coercing matched values" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" }
-        }
-      }
-    CONFIG
+    let(:config) { { "match" => { "message" => "%{NUMBER:foo:int} %{NUMBER:bar:float}" } } }
+    let(:message) { '400 454.33' }
 
-    sample "400 454.33" do
-      insist { subject.get("foo") } == 400
-      insist { subject.get("foo") }.is_a?(Integer)
-      insist { subject.get("bar") } == 454.33
-      insist { subject.get("bar") }.is_a?(Float)
+    it "coerces matched values" do
+      expect( event.get("foo") ).to be_a Integer
+      expect( event.get("foo") ).to eql 400
+      expect( event.get("bar") ).to be_a Float
+      expect( event.get("bar") ).to eql 454.33
     end
   end
 
   describe "in-line pattern definitions" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{FIZZLE=\\d+}" }
-          named_captures_only => false
-        }
-      }
-    CONFIG
+    let(:config) { { "match" => { "message" => "%{FIZZLE=\\d+}" }, "named_captures_only" => false } }
 
     sample "hello 1234" do
-      insist { subject.get("FIZZLE") } == "1234"
+      expect( event.get("FIZZLE") ).to eql '1234'
     end
   end
 
   describe "processing selected fields" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{WORD:word}" }
-          match => { "examplefield" => "%{NUMBER:num}" }
-          break_on_match => false
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{WORD:word}", "examplefield" => "%{NUMBER:num}" },
+        'break_on_match' => false
       }
-    CONFIG
+    }
+    let(:data) { { "message" => "hello world", "examplefield" => "12345" } }
 
-    sample("message" => "hello world", "examplefield" => "12345") do
-      insist { subject.get("examplefield") } == "12345"
-      insist { subject.get("word") } == "hello"
+    it "processes declared matches" do
+      expect( event.get("word") ).to eql 'hello'
+      expect( event.get("examplefield") ).to eql '12345'
     end
   end
 
   describe "adding fields on match" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "matchme %{NUMBER:fancy}" }
-          add_field => [ "new_field", "%{fancy}" ]
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "matchme %{NUMBER:fancy}" },
+        'add_field' => [ "new_field", "%{fancy}" ]
       }
-    CONFIG
+    }
 
     sample "matchme 1234" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("new_field") } == "1234"
+      expect( event.get("tags") ).to be nil
+      expect( event.get("new_field") ).to eql "1234"
     end
 
     sample "this will not be matched" do
-      insist { subject.get("tags") }.include?("_grokparsefailure")
-      reject { subject }.include?("new_field")
+      expect( event.get("tags") ).to include("_grokparsefailure")
+      expect( event ).not_to include 'new_field'
     end
   end
 
   context "empty fields" do
     describe "drop by default" do
-      config <<-CONFIG
-        filter {
-          grok {
-            match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
-          }
+      let(:config) {
+        {
+          'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
         }
-      CONFIG
+      }
 
       sample "1=test" do
-        insist { subject.get("tags") }.nil?
-        insist { subject }.include?("foo1")
+        expect( event.get("tags") ).to be nil
+        expect( event ).to include 'foo1'
 
         # Since 'foo2' was not captured, it must not be present in the event.
-        reject { subject }.include?("foo2")
+        expect( event ).not_to include 'foo2'
       end
     end
 
     describe "keep if keep_empty_captures is true" do
-      config <<-CONFIG
-        filter {
-          grok {
-            match => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" }
-            keep_empty_captures => true
-          }
+      let(:config) {
+        {
+          'match' => { "message" => "1=%{WORD:foo1} *(2=%{WORD:foo2})?" },
+          'keep_empty_captures' => true
         }
-      CONFIG
+      }
 
       sample "1=test" do
-        insist { subject.get("tags") }.nil?
+        expect( event.get("tags") ).to be nil
         # use .to_hash for this test, for now, because right now
         # the Event.include? returns false for missing fields as well
         # as for fields with nil values.
-        insist { subject.to_hash }.include?("foo2")
-        insist { subject.to_hash }.include?("foo2")
+        expect( event.to_hash ).to include 'foo1'
+        expect( event.to_hash ).to include 'foo2'
       end
     end
   end
 
   describe "when named_captures_only == false" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "Hello %{WORD}. %{WORD:foo}" }
-          named_captures_only => false
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "Hello %{WORD}. %{WORD:foo}" },
+        'named_captures_only' => false
       }
-    CONFIG
+    }
 
     sample "Hello World, yo!" do
-      insist { subject }.include?("WORD")
-      insist { subject.get("WORD") } == "World"
-      insist { subject }.include?("foo")
-      insist { subject.get("foo") } == "yo"
+      expect( event ).to include 'WORD'
+      expect( event.get("WORD") ).to eql "World"
+      expect( event ).to include 'foo'
+      expect( event.get("foo") ).to eql "yo"
     end
   end
 
   describe "using oniguruma named captures (?<name>regex)" do
     context "plain regexp" do
-      config <<-'CONFIG'
-        filter {
-          grok {
-            match => { "message" => "(?<foo>\w+)" }
-          }
+      let(:config) {
+        {
+          'match' => { "message" => "(?<foo>\\w+)" }
         }
-      CONFIG
+      }
+
       sample "hello world" do
-        insist { subject.get("tags") }.nil?
-        insist { subject.get("foo") } == "hello"
+        expect( event.get("tags") ).to be nil
+        expect( event.get("foo") ).to eql "hello"
       end
     end
 
     context "grok patterns" do
-      config <<-'CONFIG'
-        filter {
-          grok {
-            match => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
-          }
+      let(:config) {
+        {
+          'match' => { "message" => "(?<timestamp>%{DATE_EU} %{TIME})" }
         }
-      CONFIG
+      }
 
       sample "fancy 12-12-12 12:12:12" do
-        insist { subject.get("tags") }.nil?
-        insist { subject.get("timestamp") } == "12-12-12 12:12:12"
+        expect( event.get("tags") ).to be nil
+        expect( event.get("timestamp") ).to eql "12-12-12 12:12:12"
       end
     end
   end
 
   describe "grok on integer types" do
-    config <<-'CONFIG'
-      filter {
-        grok {
-          match => { "status" => "^403$" }
-          add_tag => "four_oh_three"
-        }
+    let(:config) {
+      {
+        'match' => { "status" => "^403$" }, 'add_tag' => "four_oh_three"
       }
-    CONFIG
+    }
+    let(:data) { Hash({ "status" => 403 }) }
 
-    sample("status" => 403) do
-      reject { subject.get("tags") }.include?("_grokparsefailure")
-      insist { subject.get("tags") }.include?("four_oh_three")
+    it "parses" do
+      expect( event.get("tags") ).not_to include "_grokparsefailure"
+      expect( event.get("tags") ).to include "four_oh_three"
     end
   end
 
   describe "grok on float types" do
-    config <<-'CONFIG'
-      filter {
-        grok {
-          match => { "version" => "^1.0$" }
-          add_tag => "one_point_oh"
-        }
+    let(:config) {
+      {
+        'match' => { "version" => "^1.0$" }, 'add_tag' => "one_point_oh"
       }
-    CONFIG
+    }
+    let(:data) { Hash({ "version" => 1.0 }) }
 
-    sample("version" => 1.0) do
-      insist { subject.get("tags") }.include?("one_point_oh")
-      insist { subject.get("tags") }.include?("one_point_oh")
+    it "parses" do
+      expect( event.get("tags") ).not_to include "_grokparsefailure"
+      expect( event.get("tags") ).to include "one_point_oh"
     end
   end
 
   describe "grok on %{LOGLEVEL}" do
-    config <<-'CONFIG'
-      filter {
-        grok {
-          match => { "message" => "%{LOGLEVEL:level}: error!" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{LOGLEVEL:level}: error!" }
       }
-    CONFIG
+    }
 
     log_level_names = %w(
       trace Trace TRACE
@@ -402,591 +367,528 @@ describe LogStash::Filters::Grok do
     )
     log_level_names.each do |level_name|
       sample "#{level_name}: error!" do
-        insist { subject.get("level") } == level_name
+        expect( event.get("level") ).to eql level_name
       end
     end
   end
 
   describe "timeout on failure" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => {
-            "message" => "(.*a){30}"
-          }
-          timeout_millis => 100
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "(.*a){30}" },
+        'timeout_millis' => 100
       }
-    CONFIG
+    }
 
     sample "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" do
-      expect(subject.get("tags")).to include("_groktimeout")
-      expect(subject.get("tags")).not_to include("_grokparsefailure")
+      expect( event.get("tags") ).to include("_groktimeout")
+      expect( event.get("tags") ).not_to include("_grokparsefailure")
     end
   end
 
   describe "no timeout on failure with multiple patterns (when timeout not grouped)" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => {
-            "message" => [
-              "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
-              "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
-              "(.*a){20}"
-            ]
-          }
-          timeout_millis => 500
-          timeout_scope => 'pattern'
-        }
-      }
-    CONFIG
+    let(:config) {
+      {
+        'match' => {
+          "message" => [
+            "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
+            "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
+            "(.*a){20}"
+          ]
+        },
+        'timeout_millis' => 500,
+        'timeout_scope' => 'pattern'
+      }
+    }
 
     sample( 'b' * 10 + 'c' * 10 + 'd' * 10 + 'e' * 10 + ' ' + 'a' * 20 ) do
-      insist { subject.get("tags") }.nil?
+      expect( event.get("tags") ).to be nil
     end
   end
 
   describe "timeout on grouped (multi-pattern) failure" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => {
-            "message" => [
-              "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
-              "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
-              "(.*a){20}"
-            ]
-          }
-          timeout_millis => 500
-          timeout_scope => 'event'
-        }
-      }
-    CONFIG
+    let(:config) {
+      {
+        'match' => {
+          "message" => [
+            "(.*f){20}", "(.*e){20}", "(.*d){20}", "(.*c){20}", "(.*b){20}",
+            "(.*a){25}", "(.*a){24}", "(.*a){23}", "(.*a){22}", "(.*a){21}",
+            "(.*a){20}"
+          ]
+        },
+        'timeout_millis' => 500,
+        'timeout_scope' => 'event'
+      }
+    }
 
     sample( 'b' * 10 + 'c' * 10 + 'd' * 10 + 'e' * 10 + ' ' + 'a' * 20 ) do
-      expect(subject.get("tags")).to include("_groktimeout")
-      expect(subject.get("tags")).not_to include("_grokparsefailure")
+      expect( event.get("tags") ).to include("_groktimeout")
+      expect( event.get("tags") ).not_to include("_grokparsefailure")
     end
   end
 
   describe "tagging on failure" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "matchme %{NUMBER:fancy}" }
-          tag_on_failure => not_a_match
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "matchme %{NUMBER:fancy}" },
+        'tag_on_failure' => 'not_a_match'
       }
-    CONFIG
+    }
 
     sample "matchme 1234" do
-      insist { subject.get("tags") }.nil?
+      expect( event.get("tags") ).to be nil
     end
 
     sample "this will not be matched" do
-      insist { subject.get("tags") }.include?("not_a_match")
+      expect( event.get("tags") ).to include("not_a_match")
     end
   end
 
   describe "captures named fields even if the whole text matches" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{DATE_EU:stimestamp}" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{DATE_EU:stimestamp}" }
       }
-    CONFIG
+    }
 
     sample "11/01/01" do
-      insist { subject.get("stimestamp") } == "11/01/01"
+      expect( event.get("stimestamp") ).to eql "11/01/01"
     end
   end
 
   describe "allow dashes in capture names" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{WORD:foo-bar}" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{WORD:foo-bar}" }
       }
-    CONFIG
+    }
 
     sample "hello world" do
-      insist { subject.get("foo-bar") } == "hello"
-    end
-  end
-
-  describe "performance test", :performance => true do
-    event_count = 100000
-    min_rate = 2000
-
-    max_duration = event_count / min_rate
-    input = "Nov 24 01:29:01 -0800"
-    config <<-CONFIG
-      input {
-        generator {
-          count => #{event_count}
-          message => "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
-        }
-      }
-      filter {
-        grok {
-          match => { "message" => "%{SYSLOGLINE}" }
-          overwrite => [ "message" ]
-        }
-      }
-      output { null { } }
-    CONFIG
-
-    2.times do
-      start = Time.now
-      agent do
-        duration = (Time.now - start)
-        puts "filters/grok parse rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s"
-        insist { duration } < max_duration
-      end
+      expect( event.get("foo-bar") ).to eql "hello"
     end
   end
 
   describe "single value match with duplicate-named fields in pattern" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{INT:foo}|%{WORD:foo}" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "%{INT:foo}|%{WORD:foo}" }
       }
-    CONFIG
+    }
 
     sample "hello world" do
-      insist { subject.get("foo") }.is_a?(String)
+      expect( event.get("foo") ).to be_a(String)
     end
 
     sample "123 world" do
-      insist { subject.get("foo") }.is_a?(String)
+      expect( event.get("foo") ).to be_a(String)
     end
   end
 
-  describe "break_on_match default should be true and first match should exit filter" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{INT:foo}"
-                     "somefield" => "%{INT:bar}"}
-        }
-      }
-    CONFIG
 
-    sample("message" => "hello world 123", "somefield" => "testme abc 999") do
-      insist { subject.get("foo") } == "123"
-      insist { subject.get("bar") }.nil?
-    end
-  end
-
-  describe "break_on_match when set to false should try all patterns" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{INT:foo}"
-                     "somefield" => "%{INT:bar}"}
-          break_on_match => false
-        }
+  describe "break_on_match default should be true" do
+    let(:config) {
+      {
+        'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" }
       }
-    CONFIG
+    }
+    let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
 
-    sample("message" => "hello world 123", "somefield" => "testme abc 999") do
-      insist { subject.get("foo") } == "123"
-      insist { subject.get("bar") } == "999"
+    it 'exits filter after first match' do
+      expect( event.get("foo") ).to eql '123'
+      expect( event.get("bar") ).to be nil
     end
   end
 
-  describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
-          break_on_match => false
-        }
+  describe "break_on_match when set to false" do
+    let(:config) {
+      {
+        'match' => { "message" => "%{INT:foo}", "somefield" => "%{INT:bar}" },
+        'break_on_match' => false
       }
-    CONFIG
+    }
+    let(:data) { Hash("message" => "hello world 123", "somefield" => "testme abc 999") }
 
-    sample "treebranch" do
-      insist { subject.get("name2") } == "branch"
-    end
-
-    sample "bushbeard" do
-      insist { subject.get("name1") } == "bush"
-    end
-
-    sample "treebeard" do
-      insist { subject.get("name1") } == "tree"
-      insist { subject.get("name2") } == "beard"
+    it 'should try all patterns' do
+      expect( event.get("foo") ).to eql '123'
+      expect( event.get("bar") ).to eql '999'
     end
   end
 
-  describe "break_on_match default for array input with single grok pattern" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{INT:foo}"}
-        }
+  context "break_on_match default for array input with single grok pattern" do
+    let(:config) {
+      {
+        'match' => { "message" => "%{INT:foo}" },
+        'break_on_match' => false
       }
-    CONFIG
+    }
 
-    # array input --
-    sample("message" => ["hello world 123", "line 23"]) do
-      insist { subject.get("foo") } == ["123", "23"]
-      insist { subject.get("tags") }.nil?
+    describe 'fully matching input' do
+      let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
+      it 'matches' do
+        expect( event.get("foo") ).to eql ["123", "23"]
+        expect( event.get("tags") ).to be nil
+      end
     end
 
-    # array input, one of them matches
-    sample("message" => ["hello world 123", "abc"]) do
-      insist { subject.get("foo") } == "123"
-      insist { subject.get("tags") }.nil?
+    describe 'partially matching input' do
+      let(:data) { Hash("message" => ["hello world 123", "abc"]) } # array input, one of them matches
+      it 'matches' do
+        expect( event.get("foo") ).to eql "123"
+        expect( event.get("tags") ).to be nil
+      end
     end
   end
 
   describe "break_on_match = true (default) for array input with multiple grok pattern" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
-        }
-      }
-    CONFIG
-
-    # array input --
-    sample("message" => ["hello world 123", "line 23"]) do
-      insist { subject.get("foo") } == ["123", "23"]
-      insist { subject.get("bar") }.nil?
-      insist { subject.get("tags") }.nil?
+    let(:config) {
+      {
+        'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
+      }
+    }
+
+    describe 'matching input' do
+      let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
+      it 'matches' do
+        expect( event.get("foo") ).to eql ["123", "23"]
+        expect( event.get("bar") ).to be nil
+        expect( event.get("tags") ).to be nil
+      end
     end
 
-    # array input, one of them matches
-    sample("message" => ["hello world", "line 23"]) do
-      insist { subject.get("bar") } == "hello"
-      insist { subject.get("foo") } == "23"
-      insist { subject.get("tags") }.nil?
+    describe 'partially matching input' do
+      let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
+      it 'matches' do
+        expect( event.get("bar") ).to eql 'hello'
+        expect( event.get("foo") ).to eql "23"
+        expect( event.get("tags") ).to be nil
+      end
     end
   end
 
   describe "break_on_match = false for array input with multiple grok pattern" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => ["%{INT:foo}", "%{WORD:bar}"] }
-          break_on_match => false
-        }
-      }
-    CONFIG
-
-    # array input --
-    sample("message" => ["hello world 123", "line 23"]) do
-      insist { subject.get("foo") } == ["123", "23"]
-      insist { subject.get("bar") } == ["hello", "line"]
-      insist { subject.get("tags") }.nil?
+    let(:config) {
+      {
+        'match' => { "message" => ["%{INT:foo}", "%{WORD:bar}"] },
+        'break_on_match' => false
+      }
+    }
+
+    describe 'fully matching input' do
+      let(:data) { Hash("message" => ["hello world 123", "line 23"]) } # array input --
+      it 'matches' do
+        expect( event.get("foo") ).to eql ["123", "23"]
+        expect( event.get("bar") ).to eql ["hello", "line"]
+        expect( event.get("tags") ).to be nil
+      end
     end
 
-    # array input, one of them matches
-    sample("message" => ["hello world", "line 23"]) do
-      insist { subject.get("bar") } == ["hello", "line"]
-      insist { subject.get("foo") } == "23"
-      insist { subject.get("tags") }.nil?
+    describe 'partially matching input' do
+      let(:data) { Hash("message" => ["hello world", "line 23"]) } # array input, one of them matches
+      it 'matches' do
+        expect( event.get("bar") ).to eql ["hello", "line"]
+        expect( event.get("foo") ).to eql "23"
+        expect( event.get("tags") ).to be nil
+      end
     end
   end
 
   describe  "grok with unicode" do
-    config <<-CONFIG
-      filter {
-        grok {
-          #match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
-          match => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
-        }
+    let(:config) {
+      {
+        #'match' => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+        'match' => { "message" => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}" }
       }
-    CONFIG
+    }
 
     sample "<22>Jan  4 07:50:46 mailmaster postfix/policy-spf[9454]: : SPF permerror (Junk encountered in record 'v=spf1 mx a:mail.domain.no ip4:192.168.0.4 �all'): Envelope-from: email@domain.no" do
-      insist { subject.get("tags") }.nil?
-      insist { subject.get("syslog_pri") } == "22"
-      insist { subject.get("syslog_program") } == "postfix/policy-spf"
-    end
-  end
-
-  describe "patterns in the 'patterns/' dir override core patterns" do
-
-    let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
-    let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
-
-    before do
-      FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
-      @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
-      @file.write('WORD \b[2-5]\b')
-      @file.close
-    end
-
-    let(:config) do
-      'filter { grok { match => { "message" => "%{WORD:word}" } } }'
-    end
-
-    sample("message" => 'hello') do
-      insist { subject.get("tags") } == ["_grokparsefailure"]
-    end
-
-    after do
-      File.unlink @file
-      FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
-    end
-  end
-
-  describe "patterns in custom dir override those in 'patterns/' dir" do
-
-    let(:tmpdir) { Stud::Temporary.directory }
-    let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
-    let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
-
-    before do
-      FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
-      @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
-      @file1.write('WORD \b[2-5]\b')
-      @file1.close
-      @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
-      @file2.write('WORD \b[0-1]\b')
-      @file2.close
-    end
-
-    let(:config) do
-      "filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
-    end
-
-    sample("message" => '0') do
-      insist { subject.get("tags") } == nil
-    end
-
-    after do
-      File.unlink @file1
-      File.unlink @file2
-      FileUtils.remove_entry tmpdir
-      FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
-    end
-  end
-
-  describe "patterns with file glob" do
-
-    let(:tmpdir) { Stud::Temporary.directory }
-
-    before do
-      @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
-      @file3.write('WORD \b[0-1]\b')
-      @file3.close
-      @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
-      @file4.write('WORD \b[2-5]\b')
-      @file4.close
-    end
-
-    let(:config) do
-      "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
-    end
-
-    sample("message" => '0') do
-      insist { subject.get("tags") } == nil
-    end
-
-    after do
-      File.unlink @file3
-      File.unlink @file4
-      FileUtils.remove_entry tmpdir
-    end
-  end
-
-  describe "patterns with file glob on directory that contains subdirectories" do
-
-    let(:tmpdir) { Stud::Temporary.directory }
-
-    before do
-      @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
-      @file3.write('WORD \b[0-1]\b')
-      @file3.close
-      Dir.mkdir(File.join(tmpdir, "subdir"))
-    end
-
-    let(:config) do
-      "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
-    end
-
-    sample("message" => '0') do
-      insist { subject.get("tags") } == nil
-    end
-
-    after do
-      File.unlink @file3
-      FileUtils.remove_entry tmpdir
+      expect( event.get("tags") ).to be nil
+      expect( event.get("syslog_pri") ).to eql "22"
+      expect( event.get("syslog_program") ).to eql "postfix/policy-spf"
     end
   end
 
   describe  "grok with nil coerced value" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
       }
-    CONFIG
+    }
 
     sample "test 28.4ms" do
-      insist { subject.get("duration") } == 28.4
-      insist { subject.get("tags") }.nil?
+      expect( event.get("duration") ).to eql 28.4
+      expect( event.get("tags") ).to be nil
     end
 
     sample "test N/A" do
-      insist { insist { subject.to_hash }.include?("duration") }.fails
-      insist { subject.get("tags") }.nil?
+      expect( event.to_hash ).not_to include("duration")
+      expect( event.get("tags") ).to be nil
     end
 
     sample "test abc" do
-      insist { subject.get("duration") }.nil?
-      insist { subject.get("tags") } == ["_grokparsefailure"]
+      expect( event.get("duration") ).to be nil
+      expect( event.get("tags") ).to eql ["_grokparsefailure"]
     end
   end
 
   describe  "grok with nil coerced value and keep_empty_captures" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" }
-          keep_empty_captures => true
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "test (N/A|%{BASE10NUM:duration:float}ms)" },
+        'keep_empty_captures' => true
       }
-    CONFIG
+    }
 
     sample "test N/A" do
-      insist { subject.to_hash }.include?("duration")
-      insist { subject.get("tags") }.nil?
+      expect( event.to_hash ).to include("duration")
+      expect( event.get("tags") ).to be nil
     end
-
   end
 
   describe  "grok with no coercion" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" }
-        }
+    let(:config) {
+      {
+        'match' => { "message" => "test (N/A|%{BASE10NUM:duration}ms)" },
       }
-    CONFIG
+    }
 
     sample "test 28.4ms" do
-      insist { subject.get("duration") } == "28.4"
-      insist { subject.get("tags") }.nil?
+      expect( event.get("duration") ).to eql '28.4'
+      expect( event.get("tags") ).to be nil
     end
 
     sample "test N/A" do
-      insist { subject.get("duration") }.nil?
-      insist { subject.get("tags") }.nil?
+      expect( event.get("duration") ).to be nil
+      expect( event.get("tags") ).to be nil
     end
   end
 
   describe "opening/closing" do
-    let(:config) { {"match" => {"message" => "A"}} }
-    subject(:plugin) do
-      ::LogStash::Filters::Grok.new(config)
-    end
-
-    before do
-      plugin.register
-    end
+    let(:config) { { "match" => {"message" => "A"} } }
+    let(:message) { 'AAA' }
 
     it "should close cleanly" do
-      expect { plugin.do_close }.not_to raise_error
+      expect { subject.do_close }.not_to raise_error
     end
   end
 
   describe "after grok when the event is JSON serialised the field values are unchanged" do
-    config <<-CONFIG
-      filter {grok {match => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"] remove_field => ["message","severity"] add_tag => ["ssh_failure"]}}
-    CONFIG
+    let(:config) {
+      {
+        'match' => ["message", "Failed password for (invalid user |)%{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port}"],
+        'remove_field' => ["message","severity"],
+        'add_tag' => ["ssh_failure"]
+      }
+    }
 
     sample('{"facility":"auth","message":"Failed password for testuser from 1.1.1.1 port 22"}') do
-      insist { subject.get("username") } == "testuser"
-      insist { subject.get("port") } == "22"
-      insist { subject.get("src_ip") } == "1.1.1.1"
-      insist { LogStash::Json.dump(subject.get('username')) } == "\"testuser\""
+      expect( event.get("username") ).to eql "testuser"
+      expect( event.get("port") ).to eql "22"
+      expect( event.get("src_ip") ).to eql "1.1.1.1"
+      expect( LogStash::Json.dump(event.get('username')) ).to eql "\"testuser\""
 
-      insist { subject.to_json } =~ %r|"src_ip":"1.1.1.1"|
-      insist { subject.to_json } =~ %r|"@timestamp":"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ"|
-      insist { subject.to_json } =~ %r|"port":"22"|
-      insist { subject.to_json } =~ %r|"@version":"1"|
-      insist { subject.to_json } =~ %r|"username"|i
-      insist { subject.to_json } =~ %r|"testuser"|
-      insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]|
+      expect( event.to_json ).to match %r|"src_ip":"1.1.1.1"|
+      expect( event.to_json ).to match %r|"@timestamp":"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ"|
+      expect( event.to_json ).to match %r|"port":"22"|
+      expect( event.to_json ).to match %r|"@version":"1"|
+      expect( event.to_json ).to match %r|"username"|i
+      expect( event.to_json ).to match %r|"testuser"|
+      expect( event.to_json ).to match %r|"tags":\["ssh_failure"\]|
     end
   end
 
   describe  "grok with inline pattern definition successfully extracts fields" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" }
-          pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
-           "MY_PATTERN" => "%{YEAR}"}
+    let(:config) {
+      {
+        'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" },
+        'pattern_definitions' => {
+          "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
+          "MY_PATTERN" => "%{YEAR}"
         }
       }
-    CONFIG
+    }
 
     sample "Mon Dec 26 16:22:08 2016 error 2020" do
-      insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
-      insist { subject.get("level") } == "error"
-      insist { subject.get("hindsight") } == "2020"
+      expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
+      expect( event.get("level") ).to eql "error"
+      expect( event.get("hindsight") ).to eql "2020"
     end
   end
 
   describe  "grok with inline pattern definition overwrites existing pattern definition" do
-    config <<-CONFIG
-      filter {
-        grok {
-          match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" }
-          # loglevel was previously ([Aa]lert|ALERT|[Tt]...
-          pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
-           "LOGLEVEL" => "%{NUMBER}"}
+    let(:config) {
+      {
+        'match' => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" },
+        # loglevel was previously ([Aa]lert|ALERT|[Tt]...
+        'pattern_definitions' => {
+          "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
+          "LOGLEVEL" => "%{NUMBER}"
         }
       }
-    CONFIG
+    }
 
     sample "Mon Dec 26 16:22:08 2016 9999" do
-      insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
-      insist { subject.get("level") } == "9999"
+      expect( event.get("timestamp") ).to eql "Mon Dec 26 16:22:08 2016"
+      expect( event.get("level") ).to eql "9999"
     end
   end
 
+  context 'when timeouts are explicitly disabled' do
+    let(:config) do
+      {
+        "timeout_millis" => 0
+      }
+    end
+
+    context 'when given a pathological input', slow: true do
+      let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
+      let(:config) { super().merge("match" => { "message" => "(.*a){30}" }) }
 
-  describe "direct plugin testing" do
-    subject do
-      plugin = LogStash::Filters::Grok.new(options)
-      plugin.register
-      plugin
+      it 'blocks for at least 3 seconds' do
+        blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
+        expect do
+          Timeout.timeout(3, blocking_exception_class) do
+            subject.filter(event)
+          end
+        end.to raise_exception(blocking_exception_class)
+      end
     end
+  end
+end
+
+describe LogStash::Filters::Grok do
+  describe "(LEGACY)" do
+    describe "patterns in the 'patterns/' dir override core patterns" do
 
-    let(:data) { {"message" => message} }
-    let(:event) { LogStash::Event.new(data) }
+      let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
+      let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
 
-    context 'when timeouts are explicitly disabled' do
-      let(:options) do
-        {
-          "timeout_millis" => 0
+      before do
+        FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
+        @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
+        @file.write('WORD \b[2-5]\b')
+        @file.close
+      end
+
+      let(:config) do
+        'filter { grok { match => { "message" => "%{WORD:word}" } } }'
+      end
+
+      sample("message" => 'hello') do
+        insist { subject.get("tags") } == ["_grokparsefailure"]
+      end
+
+      after do
+        File.unlink @file
+        FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
+      end
+    end
+
+    describe "patterns in custom dir override those in 'patterns/' dir" do
+
+      let(:tmpdir) { Stud::Temporary.directory }
+      let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
+      let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
+
+      before do
+        FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
+        @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
+        @file1.write('WORD \b[2-5]\b')
+        @file1.close
+        @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
+        @file2.write('WORD \b[0-1]\b')
+        @file2.close
+      end
+
+      let(:config) do
+        "filter { grok { patterns_dir => \"#{tmpdir}\" match => { \"message\" => \"%{WORD:word}\" } } }"
+      end
+
+      sample("message" => '0') do
+        insist { subject.get("tags") } == nil
+      end
+
+      after do
+        File.unlink @file1
+        File.unlink @file2
+        FileUtils.remove_entry tmpdir
+        FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
+      end
+    end
+
+    describe "patterns with file glob" do
+
+      let(:tmpdir) { Stud::Temporary.directory }
+
+      before do
+        @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
+        @file3.write('WORD \b[0-1]\b')
+        @file3.close
+        @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
+        @file4.write('WORD \b[2-5]\b')
+        @file4.close
+      end
+
+      let(:config) do
+        "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" match => { \"message\" => \"%{WORD:word}\" } } }"
+      end
+
+      sample("message" => '0') do
+        insist { subject.get("tags") } == nil
+      end
+
+      after do
+        File.unlink @file3
+        File.unlink @file4
+        FileUtils.remove_entry tmpdir
+      end
+    end
+
+    describe "patterns with file glob on directory that contains subdirectories" do
+
+      let(:tmpdir) { Stud::Temporary.directory }
+
+      before do
+        @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
+        @file3.write('WORD \b[0-1]\b')
+        @file3.close
+        Dir.mkdir(File.join(tmpdir, "subdir"))
+      end
+
+      let(:config) do
+        "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
+      end
+
+      sample("message" => '0') do
+        insist { subject.get("tags") } == nil
+      end
+
+      after do
+        File.unlink @file3
+        FileUtils.remove_entry tmpdir
+      end
+    end
+
+    describe "LOGSTASH-1547 - break_on_match should work on fields with multiple patterns" do
+      config <<-CONFIG
+      filter {
+        grok {
+          match => { "message" => ["%{GREEDYDATA:name1}beard", "tree%{GREEDYDATA:name2}"] }
+          break_on_match => false
         }
+      }
+      CONFIG
+
+      sample "treebranch" do
+        insist { subject.get("name2") } == "branch"
       end
 
-      context 'when given a pathological input' do
-        let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
-        let(:options) { super().merge("match" => { "message" => "(.*a){30}" }) }
+      sample "bushbeard" do
+        insist { subject.get("name1") } == "bush"
+      end
 
-        it 'blocks for at least 3 seconds' do
-          blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
-          expect do
-            Timeout.timeout(3, blocking_exception_class) do
-              subject.filter(event)
-            end
-          end.to raise_exception(blocking_exception_class)
-        end
+      sample "treebeard" do
+        insist { subject.get("name1") } == "tree"
+        insist { subject.get("name2") } == "beard"
       end
     end
   end