logstash-filter-kv 4.2.1 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b34ac3de7b0a2195dd497f68fe24b52acf22469467cd4fdce5052fc3c893b703
-  data.tar.gz: c8979c98ca325f26d09722000ada76d205f1a5499fcaf77e0ee5b03fd4f00088
+  metadata.gz: c8b4c710387a809508f0e21af10fbaa40ba7e5d5767ed7a104b56445c98eef97
+  data.tar.gz: 717b1f7c691c20ff4bcf0faa009dee6e15246915542872625c91d3499a51910a
 SHA512:
-  metadata.gz: 91ba5899a87d934d7bd0b19a2fc18f0c6ea8dabd01bb093e558536afcf53a1568cbe72035b1c61e4cb071c7cdd4c2b9d8195fa7fe4d2d3f8bec91aae2ec99979
-  data.tar.gz: feef4f88bd22e8e3486e583fdab40d1b3e5e70590f704436579790f464afcf76de183ba49a57502d915fab2cc1358f0faaf215c3bf5759deb882879aad9b7b09
+  metadata.gz: 732251129f69923ea68133b38d6aa5e346b67d124903f3c3e98a113f6b508aebe498fbe1329b4dc44589600d8423a442331b70cc4d8dfe6d51cbbd3c93bf58fd
+  data.tar.gz: 9f348a9d4318ab77337292b8fbfe17d717cce0bd2e192733f835cd943bc21a00ccb672ba26ec812470b77a8d620caa3a922b9056cf6e1f9b82389e7936d82bb7

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.3.0
+ - Added a timeout enforcer which prevents inputs that are pathological against the generated parser from blocking
+   the pipeline. By default, timeout is a generous 30s, but can be configured or disabled entirely with the new
+   `timeout_millis` and `tag_on_timeout` directives ([#79](https://github.com/logstash-plugins/logstash-filter-kv/pull/79))
+ - Made error-handling configurable with `tag_on_failure` directive.
+
 ## 4.2.1
  - Fixes performance regression introduced in 4.1.0 ([#70](https://github.com/logstash-plugins/logstash-filter-kv/issues/70))
 

data/docs/index.asciidoc

@@ -65,6 +65,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-remove_char_value>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-source>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-tag_on_timeout>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-timeout_millis>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-transform_key>> |<<string,string>>, one of `["lowercase", "uppercase", "capitalize"]`|No
 | <<plugins-{type}s-{plugin}-transform_value>> |<<string,string>>, one of `["lowercase", "uppercase", "capitalize"]`|No
 | <<plugins-{type}s-{plugin}-trim_key>> |<<string,string>>|No
@@ -335,6 +338,38 @@ For example, to place all keys into the event field kv:
 [source,ruby]
     filter { kv { target => "kv" } }
 
+[id="plugins-{type}s-{plugin}-tag_on_failure"]
+===== `tag_on_failure`
+
+  * Value type is <<string,string>>
+  * The default value for this setting is `_kv_filter_error`.
+
+When a kv operation causes a runtime exception to be thrown within the plugin,
+the operation is safely aborted without crashing the plugin, and the event is
+tagged with the provided value.
+
+[id="plugins-{type}s-{plugin}-tag_on_timeout"]
+===== `tag_on_timeout`
+
+  * Value type is <<string,string>>
+  * The default value for this setting is `_kv_filter_timeout`.
+
+When timeouts are enabled and a kv operation is aborted, the event is tagged
+with the provided value (see: <<plugins-{type}s-{plugin}-timeout_millis>>).
+
+[id="plugins-{type}s-{plugin}-timeout_millis"]
+===== `timeout_millis`
+
+  * Value type is <<number, number>>
+  * The default value for this setting is 30000 (30 seconds).
+  * Set to zero (`0`) to disable timeouts
+
+Timeouts provide a safeguard against inputs that are pathological against the
+regular expressions that are used to extract key/value pairs. When parsing an
+event exceeds this threshold the operation is aborted and the event is tagged
+in order to prevent the operation from blocking the pipeline
+(see: <<plugins-{type}s-{plugin}-tag_on_timeout>>).
+
 [id="plugins-{type}s-{plugin}-transform_key"]
 ===== `transform_key` 
 

data/lib/logstash/filters/kv.rb

@@ -317,6 +317,19 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #
   config :whitespace, :validate => %w(strict lenient), :default => "lenient"
 
+  # Attempt to terminate regexps after this amount of time.
+  # This applies per source field value if event has multiple values in the source field.
+  # This will never timeout early, but may take a little longer to timeout.
+  # Actual timeout is approximate based on a 250ms quantization.
+  # Set to 0 to disable timeouts
+  config :timeout_millis, :validate => :number, :default => 30_000
+
+  # Tag to apply if a kv regexp times out.
+  config :tag_on_timeout, :validate => :string, :default => '_kv_filter_timeout'
+
+  # Tag to apply if kv errors
+  config :tag_on_failure, :validate => :string, :default => '_kv_filter_error'
+
   def register
     if @value_split.empty?
       raise LogStash::ConfigurationError, I18n.t(
@@ -392,21 +405,26 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     @value_split_re = value_split_pattern
 
     @logger.debug? && @logger.debug("KV scan regex", :regex => @scan_re.inspect)
+
+    @timeout_enforcer = initialize_timeout_enforcer
+    @timeout_enforcer.start!
   end
 
   def filter(event)
     kv = Hash.new
     value = event.get(@source)
 
-    case value
-    when nil
-      # Nothing to do
-    when String
-      kv = parse(value, event, kv)
-    when Array
-      value.each { |v| kv = parse(v, event, kv) }
-    else
-      @logger.warn("kv filter has no support for this type of data", :type => value.class, :value => value)
+    @timeout_enforcer.execute do
+      case value
+      when nil
+        # Nothing to do
+      when String
+        parse(value, event, kv)
+      when Array
+        value.each { |v| parse(v, event, kv) }
+      else
+        @logger.warn("kv filter has no support for this type of data", :type => value.class, :value => value)
+      end
     end
 
     # Add default key-values for missing keys
@@ -422,15 +440,47 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     end
 
     filter_matched(event)
+
+  rescue TimeoutException => e
+    logger.warn("Timeout reached in KV filter with value #{summarize(value)}")
+    event.tag(@tag_on_timeout)
   rescue => ex
     meta = { :exception => ex.message }
     meta[:backtrace] = ex.backtrace if logger.debug?
     logger.warn('Exception while parsing KV', meta)
-    event.tag('_kv_filter_error')
+    event.tag(@tag_on_failure)
+  end
+
+  def close
+    @timeout_enforcer.stop!
   end
 
   private
 
+  # @overload summarize(value)
+  #   @param value [Array]
+  #   @return [String]
+  # @overload summarize(value)
+  #   @param value [String]
+  #   @return [String]
+  def summarize(value)
+    if value.kind_of?(Array)
+      value.map(&:to_s).map do |entry|
+        summarize(entry)
+      end.to_s
+    end
+
+    value = value.to_s
+
+    value.bytesize < 255 ? "`#{value}`" : "entry too large; first 255 chars are `#{value[0..255].dump}`"
+  end
+
+  def initialize_timeout_enforcer
+    return NULL_TIMEOUT_ENFORCER if @timeout_millis <= 0
+
+    TimeoutEnforcer.new(logger, @timeout_millis * 1_000_000)
+  end
+
   def has_value_splitter?(s)
     s =~ @value_split_re
   end
@@ -487,9 +537,16 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     end
   end
 
+  # Parses the given `text`, using the `event` for context, into the provided `kv_keys` hash
+  #
+  # @param text [String]: the text to parse
+  # @param event [LogStash::Event]: the event from which to extract context (e.g., sprintf vs (in|ex)clude keys)
+  # @param kv_keys [Hash{String=>Object}]: the hash in which to inject found key/value pairs
+  #
+  # @return [void]
   def parse(text, event, kv_keys)
     # short circuit parsing if the text does not contain the @value_split
-    return kv_keys unless has_value_splitter?(text)
+    return unless has_value_splitter?(text)
 
     # Interpret dynamic keys for @include_keys and @exclude_keys
     include_keys = @include_keys.map{|key| event.sprintf(key)}
@@ -520,7 +577,8 @@ class LogStash::Filters::KV < LogStash::Filters::Base
 
       # recursively get more kv pairs from the value
       if @recursive
-        innerKv = parse(value, event, {})
+        innerKv = {}
+        parse(value, event, innerKv)
         value = innerKv unless innerKv.empty?
       end
 
@@ -534,7 +592,95 @@ class LogStash::Filters::KV < LogStash::Filters::Base
         kv_keys[key] = value
       end
     end
+  end
+
+  class TimeoutException < RuntimeError
+  end
+
+  class TimeoutEnforcer
+    def initialize(logger, timeout_nanos)
+      @logger = logger
+      @running = java.util.concurrent.atomic.AtomicBoolean.new(false)
+      @timeout_nanos = timeout_nanos
 
-    return kv_keys
+      # Stores running matches with their start time, this is used to cancel long running matches
+      # Is a map of Thread => start_time
+      @threads_to_start_time = java.util.concurrent.ConcurrentHashMap.new
+    end
+
+    def execute(&block)
+      begin
+        thread = java.lang.Thread.currentThread()
+        @threads_to_start_time.put(thread, java.lang.System.nanoTime)
+
+        yield
+
+      rescue InterruptedRegexpError, java.lang.InterruptedException => e
+        raise TimeoutException.new
+      ensure
+        # If the block finished, but interrupt was called after, we'll want to
+        # clear the interrupted status anyway
+        @threads_to_start_time.remove(thread)
+        thread.interrupted
+      end
+    end
+
+    def start!
+      @running.set(true)
+      @logger.debug("Starting timeout enforcer (#{@timeout_nanos}ns)")
+      @timer_thread = Thread.new do
+        while @running.get()
+          begin
+            cancel_timed_out!
+          rescue Exception => e
+            @logger.error("Error while attempting to check/cancel excessively long kv patterns",
+                          :message => e.message,
+                          :class => e.class.name,
+                          :backtrace => e.backtrace
+            )
+          end
+          sleep 0.25
+        end
+      end
+    end
+
+    def stop!
+      @running.set(false)
+      @logger.debug("Shutting down timeout enforcer")
+      # Check for the thread mostly for a fast start/shutdown scenario
+      @timer_thread.join if @timer_thread
+    end
+
+    private
+
+    def cancel_timed_out!
+      now = java.lang.System.nanoTime # save ourselves some nanotime calls
+      @threads_to_start_time.keySet.each do |thread|
+        # Use compute to lock this value
+        @threads_to_start_time.computeIfPresent(thread) do |thread, start_time|
+          if start_time < now && now - start_time > @timeout_nanos
+            thread.interrupt
+            nil # Delete the key
+          else
+            start_time # preserve the key
+          end
+        end
+      end
+    end
+  end
+
+  class NullTimeoutEnforcer
+    def execute(&block)
+      yield
+    end
+
+    def start!
+      # no-op
+    end
+
+    def stop!
+      # no-op
+    end
   end
+  NULL_TIMEOUT_ENFORCER = NullTimeoutEnforcer.new
 end

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '4.2.1'
+  s.version         = '4.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses key-value pairs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/kv_spec.rb

@@ -1093,5 +1093,131 @@ context 'runtime errors' do
 
       plugin.filter(event)
     end
+    context 'when a custom tag is defined' do
+      let(:options) { super().merge("tag_on_failure" => "KV-ERROR")}
+      it 'tags the event with the custom tag' do
+        plugin.filter(event)
+        expect(event.get('tags')).to_not be_nil
+        expect(event.get('tags')).to include('KV-ERROR')
+      end
+    end
+  end
+end
+
+# This group intentionally uses patterns that are vulnerable to pathological inputs to test timeouts.
+#
+# patterns of the form `/(?:x+x+)+y/` are vulnerable to inputs that have long sequences matching `/x/`
+# that are _not_ followed by a sequence matching `/y/`.
+context 'timeouts' do
+  let(:options) do
+    {
+        "value_split_pattern" => "(?:=+=+)+:"
+    }
+  end
+  subject(:plugin) do
+    LogStash::Filters::KV.new(options).instance_exec { register; self }
+  end
+
+  let(:data) { {"message" => message} }
+  let(:event) { LogStash::Event.new(data) }
+  let(:message) { "foo=bar hello=world" }
+
+  after(:each) { plugin.close }
+
+  # since we are dealing with potentially-pathological specs, ensure specs fail in a timely
+  # manner if they block for longer than `spec_blocking_threshold_seconds`.
+  let(:spec_blocking_threshold_seconds) { 10 }
+  around(:each) do |example|
+    begin
+      blocking_exception_class = Class.new(::Exception) # avoid RuntimeError, which is handled in KV#filter
+      Timeout.timeout(spec_blocking_threshold_seconds, blocking_exception_class, &example)
+    rescue blocking_exception_class
+      fail('execution blocked')
+    end
+  end
+
+  context 'when timeouts are enabled' do
+    let(:options) { super().merge("timeout_millis" => 250) }
+    let(:spec_blocking_threshold_seconds) { 3 }
+
+    context 'when given a pathological input' do
+      let(:message) { "foo========:bar baz================================================bingo" }
+
+      it 'tags the event' do
+        plugin.filter(event)
+
+        expect(event.get('tags')).to be_a_kind_of(Enumerable)
+        expect(event.get('tags')).to include('_kv_filter_timeout')
+      end
+
+      context 'when given a custom `tag_on_timeout`' do
+        let(:options) { super().merge('tag_on_timeout' => 'BADKV') }
+
+        it 'tags the event with the custom tag' do
+          plugin.filter(event)
+
+          expect(event.get('tags')).to be_a_kind_of(Enumerable)
+          expect(event.get('tags')).to include('BADKV')
+        end
+      end
+
+      context 'when default_keys are provided' do
+        let(:options) { super().merge("default_keys" => {"default" => "key"})}
+
+        it 'does not populate default keys' do
+          plugin.filter(event)
+
+          expect(event).to_not include('default')
+        end
+      end
+      context 'when filter_matched hooks are provided' do
+        let(:options) { super().merge("add_field" => {"kv" => "success"})}
+
+        it 'does not call filter_matched hooks' do
+          plugin.filter(event)
+
+          expect(event).to_not include('kv')
+        end
+      end
+    end
+
+    context 'when given a non-pathological input' do
+      let(:message) { "foo==:bar baz==:bingo" }
+
+      it 'extracts the k/v' do
+        plugin.filter(event)
+
+        expect(event.get('foo')).to eq('bar')
+        expect(event.get('baz')).to eq('bingo')
+      end
+    end
+  end
+
+  context 'when timeouts are explicitly disabled' do
+    let(:options) { super().merge("timeout_millis" => 0) }
+
+    context 'when given a pathological input' do
+      let(:message) { "foo========:bar baz================================================================bingo"}
+
+      it 'blocks for at least 3 seconds' do
+        blocking_exception_class = Class.new(::Exception) # avoid RuntimeError, which is handled in KV#filter
+        expect do
+          Timeout.timeout(3, blocking_exception_class) do
+            plugin.filter(event)
+          end
+        end.to raise_exception(blocking_exception_class)
+      end
+    end
+
+    context 'when given a non-pathological input' do
+      let(:message) { "foo==:bar baz==:bingo" }
+
+      it 'extracts the k/v' do
+        plugin.filter(event)
+
+        expect(event.get('foo')).to eq('bar')
+        expect(event.get('baz')).to eq('bingo')
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 4.2.1
+  version: 4.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-09-04 00:00:00.000000000 Z
+date: 2019-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement