logstash-filter-kv 4.1.2 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e108f33865f7afc5f0b397d3ba6ae37b93cf664c6e8f6a8d9bf74650c987954c
-  data.tar.gz: aded17fb29c331fe4d68ae280d2ed884e968ae180e6539a313c66b8399952be4
+  metadata.gz: 60449ebe2807038eecb8421ac199171dbdb021c5ed8207d5fa37ffa5904c27eb
+  data.tar.gz: 9f9b20456f61eae245a69cdcad9adc86f917711d14cd96cc1e8f8d3317caf038
 SHA512:
-  metadata.gz: e122414de884300d596a3d7da38f6408e1a1a42d32894013dd8c8fea533bcf18ac90619cc0ca544c68c552ab0a8e1c8891d1d15d2ee60aeb4c028b7f396443be
-  data.tar.gz: d667872a513d33403b7b26e69a876787ae985c515dac645948f3536f67d7b04175ec727a7f76b97d804b3934f0ee8699cd7d849a1b93f692b3c067647331ed39
+  metadata.gz: ee5f0e5a66cf9a88d70d757e7b729636a4f600d9db8022e0eeabd1220ff80065e62221dca5941efe38fdf9cb059eef026fba663a1c045ff24008c98bf985dfd2
+  data.tar.gz: 5894f8dfe7231d2a77e6a262f0422be1768613147d8a27cb565a08474dab95ec9026405f4fdbd76683ac7011271464361f98e7f3e65b65e358b81a97e9c4ba7d

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.2.0
+ - Added `whitespace => strict` mode, which allows the parser to behave more predictably when input is known to avoid unnecessary whitespace.
+ - Added error handling, which tags the event with `_kv_filter_error` if an exception is raised while handling an event instead of allowing the plugin to crash.
+
 ## 4.1.2
   - bugfix: improves trim_key and trim_value to trim any _sequence_ of matching characters from the beginning and ends of the corresponding keys and values; a previous implementation limitited trim to a single character from each end, which was surprising.
   - bugfix: fixes issue where we can fail to correctly break up a sequence that includes a partially-quoted value followed by another fully-quoted value by slightly reducing greediness of quoted-value captures.

data/docs/index.asciidoc

@@ -71,6 +71,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-trim_value>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-value_split>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-value_split_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-whitespace>> |<<string,string>>, one of `["strict", "lenient"]`|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -444,6 +445,23 @@ cautious with lookaheads or lookbehinds and positional anchors.
 
 See `field_split_pattern` for examples.
 
+[id="plugins-{type}s-{plugin}-whitespace"]
+===== `whitespace`
+
+  * Value can be any of: `lenient`, `strict`
+  * Default value is `lenient`
+
+An option specifying whether to be _lenient_ or _strict_ with the acceptance of unnecessary
+whitespace surrounding the configured value-split sequence.
+
+By default the plugin is run in `lenient` mode, which ignores spaces that occur before or
+after the value-splitter. While this allows the plugin to make reasonable guesses with most
+input, in some situations it may be too lenient.
+
+You may want to enable `whitespace => strict` mode if you have control of the input data and
+can guarantee that no extra spaces are added surrounding the pattern you have defined for
+splitting values. Doing so will ensure that a _field-splitter_ sequence immediately following
+a _value-splitter_ will be interpreted as an empty field.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/kv.rb

@@ -303,6 +303,20 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #
   config :recursive, :validate => :boolean, :default => false
 
+  # An option specifying whether to be _lenient_ or _strict_ with the acceptance of unnecessary
+  # whitespace surrounding the configured value-split sequence.
+  #
+  # By default the plugin is run in `lenient` mode, which ignores spaces that occur before or
+  # after the value-splitter. While this allows the plugin to make reasonable guesses with most
+  # input, in some situations it may be too lenient.
+  #
+  # You may want to enable `whitespace => strict` mode if you have control of the input data and
+  # can guarantee that no extra spaces are added surrounding the pattern you have defined for
+  # splitting values. Doing so will ensure that a _field-splitter_ sequence immediately following
+  # a _value-splitter_ will be interpreted as an empty field.
+  #
+  config :whitespace, :validate => %w(strict lenient), :default => "lenient"
+
   def register
     if @value_split.empty?
       raise LogStash::ConfigurationError, I18n.t(
@@ -337,12 +351,21 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     @remove_char_value_re = Regexp.new("[#{@remove_char_value}]") if @remove_char_value
     @remove_char_key_re = Regexp.new("[#{@remove_char_key}]") if @remove_char_key
 
-    field_split = Regexp::compile(@field_split_pattern || /[#{@field_split}]/)
-    value_split = Regexp::compile(@value_split_pattern || /[#{@value_split}]/)
-
-    optional_whitespace = /\s*/
+    optional_whitespace = / */
     eof = /$/
 
+    field_split_pattern = Regexp::compile(@field_split_pattern || "[#{@field_split}]")
+    value_split_pattern = Regexp::compile(@value_split_pattern || "[#{@value_split}]")
+
+    # in legacy-compatible lenient mode, the value splitter can be wrapped in optional whitespace
+    if @whitespace == 'lenient'
+      value_split_pattern = /#{optional_whitespace}#{value_split_pattern}#{optional_whitespace}/
+    end
+
+    # a key is a _captured_ sequence of characters or escaped spaces before optional whitespace
+    # and followed by either a `value_split`, a `field_split`, or EOF.
+    key_pattern = unquoted_capture(value_split_pattern, field_split_pattern, eof)
+
     value_pattern = begin
       # each component expression within value_pattern _must_ capture exactly once.
       value_patterns = []
@@ -356,18 +379,13 @@ class LogStash::Filters::KV < LogStash::Filters::Base
       end
 
       # an unquoted value is a _captured_ sequence of characters or escaped spaces before a `field_split` or EOF.
-      value_patterns << /((?:\\ |.)*?)(?=#{Regexp::union(field_split, eof)})/
+      value_patterns << unquoted_capture(field_split_pattern, eof)
 
-      Regexp.union(*value_patterns)
+      Regexp.union(value_patterns)
     end
 
-
-    # a key is a _captured_ sequence of characters or escaped spaces before optional whitespace
-    # and followed by either a `value_split`, a `field_split`, or EOF.
-    key_pattern = /((?:\\ |.)+?)(?=#{optional_whitespace}#{Regexp::union(value_split, field_split, eof)})/
-
-    @scan_re = /#{field_split}?#{key_pattern}#{optional_whitespace}(?:#{value_split}#{optional_whitespace}#{value_pattern})?(?=#{Regexp::union(field_split, eof)})/
-    @value_split_re = value_split
+    @scan_re = /#{key_pattern}#{value_split_pattern}#{value_pattern}#{Regexp::union(field_split_pattern, eof)}/
+    @value_split_re = value_split_pattern
 
     @logger.debug? && @logger.debug("KV scan regex", :regex => @scan_re.inspect)
   end
@@ -400,6 +418,11 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     end
 
     filter_matched(event)
+  rescue => ex
+    meta = { :exception => ex.message }
+    meta[:backtrace] = ex.backtrace if logger.debug?
+    logger.warn('Exception while parsing KV', meta)
+    event.tag('_kv_filter_error')
   end
 
   private
@@ -423,9 +446,19 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     close_pattern = /#{Regexp.quote(close_quote_sequence)}/
 
     # matches a sequence of zero or more characters are _not_ the `close_quote_sequence`
-    quoted_value_pattern = /[^#{Regexp.quote(close_quote_sequence)}]*/
+    quoted_value_pattern = unquoted_capture(close_pattern)
 
-    /#{open_pattern}(#{quoted_value_pattern})#{close_pattern}/
+    /#{open_pattern}#{quoted_value_pattern}?#{close_pattern}/
+  end
+
+  # Helper function for generating *capturing* `Regexp` that will match any sequence of characters that are either
+  # backslash-escaped OR *NOT* matching any of the given pattern(s)
+  #
+  # @api private
+  # @param *until_lookahead_patterns [Regexp]
+  # @return [Regexp]
+  def unquoted_capture(*until_lookahead_patterns)
+    /((?:\\.|(?!#{Regexp::union(until_lookahead_patterns)}).)+)/
   end
 
   def transform(text, method)

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '4.1.2'
+  s.version         = '4.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses key-value pairs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/kv_spec.rb

@@ -46,6 +46,48 @@ describe LogStash::Filters::KV do
     end
   end
 
+  describe 'whitespace => strict' do
+    config <<-CONFIG
+      filter {
+        kv {
+          whitespace => strict
+        }
+      }
+    CONFIG
+
+    context 'unquoted values' do
+      sample "IN=eth0 OUT= MAC=0f:5f:5e:aa:d3:a2:21:ff:09:00:0f:e1:c8:17 SRC=192.168.0.1" do
+        insist { subject.get('IN') } == 'eth0'
+        insist { subject.get('OUT') } == nil # when whitespace is strict, OUT is empty and thus uncaptured.
+        insist { subject.get('MAC') } == '0f:5f:5e:aa:d3:a2:21:ff:09:00:0f:e1:c8:17'
+        insist { subject.get('SRC') } == '192.168.0.1'
+      end
+    end
+
+    context 'mixed quotations' do
+      sample 'hello=world goodbye=cruel\\ world empty_quoted="" quoted="value1" empty_unquoted= unquoted=value2 empty_bracketed=[] bracketed=[value3] cake=delicious' do
+        insist { subject.get('hello') } == 'world'
+        insist { subject.get('goodbye') } == 'cruel\\ world'
+        insist { subject.get('empty_quoted') } == nil
+        insist { subject.get('quoted') } == 'value1'
+        insist { subject.get('empty_unquoted') } == nil
+        insist { subject.get('unquoted') } == 'value2'
+        insist { subject.get('empty_bracketed') } == nil
+        insist { subject.get('bracketed') } == 'value3'
+        insist { subject.get('cake') } == 'delicious'
+      end
+    end
+
+    context 'when given sloppy input, it extracts only the unambiguous bits' do
+      sample "hello = world foo =bar baz= fizz whitespace=none doublequoted = \"hello world\" singlequoted= 'hello world' brackets =(hello world) strict=true" do
+        insist { subject.get('whitespace') } == 'none'
+        insist { subject.get('strict') } == 'true'
+
+        insist { subject.to_hash.keys.sort } == %w(@timestamp @version message strict whitespace)
+      end
+    end
+  end
+
   describe  "test transforming keys to lowercase and values to uppercase" do
     config <<-CONFIG
       filter {
@@ -1018,3 +1060,38 @@ describe "multi character splitting" do
     end
   end
 end
+
+context 'runtime errors' do
+
+  let(:options) { {} }
+  let(:plugin) do
+    LogStash::Filters::KV.new(options).instance_exec { register; self }
+  end
+
+  let(:data) { {"message" => message} }
+  let(:event) { LogStash::Event.new(data) }
+  let(:message) { "foo=bar hello=world" }
+
+
+  before(:each) do
+    expect(plugin).to receive(:parse) { fail('intentional') }
+  end
+
+  context 'when a runtime error is raised' do
+    it 'does not cascade the exception to crash the plugin' do
+      plugin.filter(event)
+    end
+    it 'tags the event with "_kv_filter_error"' do
+      plugin.filter(event)
+      expect(event.get('tags')).to_not be_nil
+      expect(event.get('tags')).to include('_kv_filter_error')
+    end
+    it 'logs an informative message' do
+      logger_double = double('Logger').as_null_object
+      expect(plugin).to receive(:logger).and_return(logger_double).at_least(:once)
+      expect(logger_double).to receive(:warn).with('Exception while parsing KV', anything)
+
+      plugin.filter(event)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 4.1.2
+  version: 4.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-16 00:00:00.000000000 Z
+date: 2018-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement