logstash-filter-kv 4.1.1 → 4.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c2b2e468b13494eda3ab5a4252ec1bc3b1ac73516c7de0f767d5f3a21d1c467
-  data.tar.gz: 145252f085544e2edbceef88a6bef4cc720f6ce745e022150c25657588c3f6d8
+  metadata.gz: e108f33865f7afc5f0b397d3ba6ae37b93cf664c6e8f6a8d9bf74650c987954c
+  data.tar.gz: aded17fb29c331fe4d68ae280d2ed884e968ae180e6539a313c66b8399952be4
 SHA512:
-  metadata.gz: 718f14bae4895dd2d9cfc252abaace7ae1f784703ec569fb9143946d21378d638889dca25db6f430bdaa25f7838eedda6c2e08eab76d520385701b7171752bcb
-  data.tar.gz: acee9e93136b19746cac5714cf46d5d45e0780caba1ab935aa42a1f26baaa45a111b5214b04cb7dff6499e11a2f6011053d7a46180f8c9b6d78419d80f5b2e13
+  metadata.gz: e122414de884300d596a3d7da38f6408e1a1a42d32894013dd8c8fea533bcf18ac90619cc0ca544c68c552ab0a8e1c8891d1d15d2ee60aeb4c028b7f396443be
+  data.tar.gz: d667872a513d33403b7b26e69a876787ae985c515dac645948f3536f67d7b04175ec727a7f76b97d804b3934f0ee8699cd7d849a1b93f692b3c067647331ed39

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.1.2
+  - bugfix: improves trim_key and trim_value to trim any _sequence_ of matching characters from the beginning and ends of the corresponding keys and values; a previous implementation limitited trim to a single character from each end, which was surprising.
+  - bugfix: fixes issue where we can fail to correctly break up a sequence that includes a partially-quoted value followed by another fully-quoted value by slightly reducing greediness of quoted-value captures.
+
 ## 4.1.1
   - bugfix: correctly handle empty values between value separator and field separator (#58)
 

data/docs/index.asciidoc

@@ -56,6 +56,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-default_keys>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-exclude_keys>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-field_split>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-field_split_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-include_brackets>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-include_keys>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
@@ -69,6 +70,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-trim_key>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-trim_value>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-value_split>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-value_split_pattern>> |<<string,string>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -444,4 +446,4 @@ See `field_split_pattern` for examples.
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/kv.rb

@@ -331,8 +331,8 @@ class LogStash::Filters::KV < LogStash::Filters::Base
       )
     end
 
-    @trim_value_re = Regexp.new("^[#{@trim_value}]|[#{@trim_value}]$") if @trim_value
-    @trim_key_re = Regexp.new("^[#{@trim_key}]|[#{@trim_key}]$") if @trim_key
+    @trim_value_re = Regexp.new("^[#{@trim_value}]+|[#{@trim_value}]+$") if @trim_value
+    @trim_key_re = Regexp.new("^[#{@trim_key}]+|[#{@trim_key}]+$") if @trim_key
 
     @remove_char_value_re = Regexp.new("[#{@remove_char_value}]") if @remove_char_value
     @remove_char_key_re = Regexp.new("[#{@remove_char_key}]") if @remove_char_key
@@ -422,8 +422,8 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     open_pattern = /#{Regexp.quote(quote_sequence)}/
     close_pattern = /#{Regexp.quote(close_quote_sequence)}/
 
-    # matches a sequence of zero or more characters that is followed by the `close_quote_sequence`
-    quoted_value_pattern = /(?:.)*?(?=#{Regexp.quote(close_quote_sequence)})/
+    # matches a sequence of zero or more characters are _not_ the `close_quote_sequence`
+    quoted_value_pattern = /[^#{Regexp.quote(close_quote_sequence)}]*/
 
     /#{open_pattern}(#{quoted_value_pattern})#{close_pattern}/
   end

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '4.1.1'
+  s.version         = '4.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses key-value pairs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/kv_spec.rb

@@ -730,6 +730,59 @@ describe LogStash::Filters::KV do
     end
   end
 
+  describe "trim_key/trim_value options : trim multiple matching characters from either end" do
+    subject do
+      plugin = LogStash::Filters::KV.new(options)
+      plugin.register
+      plugin
+    end
+
+    let(:data) { {"message" => message} }
+    let(:event) { LogStash::Event.new(data) }
+
+
+    context 'repeated same-character sequence' do
+      let(:message) { "key1=  value1 with spaces    |  key2 with spaces  =value2" }
+      let(:options) {
+        {
+            "field_split" => "|",
+            "value_split" => "=",
+            "trim_value" => " ",
+            "trim_key" => " "
+        }
+      }
+
+      it 'trims all the right bits' do
+        subject.filter(event)
+        expect(event.get('key1')).to eq('value1 with spaces')
+        expect(event.get('key2 with spaces')).to eq('value2')
+      end
+    end
+
+    context 'multi-character sequence' do
+      let(:message) { "to=<foo@example.com>, orig_to=<bar@example.com>, %+relay=mail.example.com[private/dovecot-lmtp], delay=2.2, delays=1.9/0.01/0.01/0.21, dsn=2.0.0, status=sent (250 2.0.0 <foo@example.com> YERDHejiRSXFDSdfUXTV Saved) " }
+      let(:options) {
+        {
+            "field_split" => " ",
+            "value_split" => "=",
+            "trim_value" => "<>,",
+            "trim_key" => "%+"
+        }
+      }
+
+      it 'trims all the right bits' do
+        subject.filter(event)
+        expect(event.get('to')).to eq('foo@example.com')
+        expect(event.get('orig_to')).to eq('bar@example.com')
+        expect(event.get('relay')).to eq('mail.example.com[private/dovecot-lmtp]')
+        expect(event.get('delay')).to eq('2.2')
+        expect(event.get('delays')).to eq('1.9/0.01/0.01/0.21')
+        expect(event.get('dsn')).to eq('2.0.0')
+        expect(event.get('status')).to eq('sent')
+      end
+    end
+  end
+
   describe "remove_char_key/remove_char_value options : remove all characters in keys/values whatever their position" do
     subject do
       plugin = LogStash::Filters::KV.new(options)
@@ -833,6 +886,40 @@ describe "multi character splitting" do
     it_behaves_like "parsing all fields and values"
   end
 
+  context 'multi-char field split pattern with value that begins quoted and contains more unquoted' do
+    let(:message) { 'foo=bar!!!!!baz="quoted stuff" and more unquoted!!!!!msg="fully-quoted with a part! of the separator"!!!!!blip="this!!!!!is it"!!!!!empty=""!!!!!non-empty="foo"' }
+    let(:options) {
+      {
+          "field_split_pattern" => "!!!!!"
+      }
+    }
+    it 'gets the right bits' do
+      subject.filter(event)
+      expect(event.get("foo")).to eq('bar')
+      expect(event.get("baz")).to eq('"quoted stuff" and more unquoted')
+      expect(event.get("msg")).to eq('fully-quoted with a part! of the separator')
+      expect(event.get("blip")).to eq('this!!!!!is it')
+      expect(event.get("empty")).to be_nil
+      expect(event.get("non-empty")).to eq('foo')
+    end
+  end
+
+  context 'standard field split pattern with value that begins quoted and contains more unquoted' do
+    let(:message) { 'foo=bar baz="quoted stuff" and more unquoted msg="some fully-quoted message " empty="" non-empty="foo"' }
+    let(:options) {
+      {
+      }
+    }
+    it 'gets the right bits' do
+      subject.filter(event)
+      expect(event.get("foo")).to eq('bar')
+      expect(event.get("baz")).to eq('quoted stuff') # NOTE: outside the quotes is truncated because field split pattern wins.
+      expect(event.get("msg")).to eq('some fully-quoted message ')
+      expect(event.get("empty")).to be_nil
+      expect(event.get("non-empty")).to eq('foo')
+    end
+  end
+
   context "field and value split multi" do
     let(:message) { "hello::world__foo::bar__baz::fizz__doublequoted::\"hello world\"__singlequoted::'hello world'__bracketsone::(hello world)__bracketstwo::[hello world]__bracketsthree::<hello world>" }
     let(:options) {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.1.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-23 00:00:00.000000000 Z
+date: 2018-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement