logstash-filter-kv 3.1.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39ddb76c9ee42cbde4072495a4e95a11d9b2869f
-  data.tar.gz: 37a419b9ab32acb5b98aca490cdc8a33ce30fe80
+  metadata.gz: bd187e13f268bccf3d773032be91182db9539218
+  data.tar.gz: 3869c2cd4527da8298067f4558d809b7c034e87d
 SHA512:
-  metadata.gz: d3f8b3dc0f03ea34047b5ca28c6911c5c2a9f500aa8bf711805981415e31996e8356144618dd237a756f76ec781bae83edd1a59aa3e48e689fc387b61b8e805e
-  data.tar.gz: 043117582b6b089324676523d5f22090ddc50647662433ad8745c84f9211311c00659e8df7a171c92374047a011dc10b1db1c29ec726f0f4e99212b4360afdf5
+  metadata.gz: f6d25cd2beb66fad753125e9f0f49806dbf908771ab7ef1ec73584feb5538b36857cec4ba5e6f75df30aecab87d5ee2b573d6c60b9b22096a3cc27484859e42f
+  data.tar.gz: 905f6b2f0fc3ab89dfe95a3cfbd2b5e76a7f50835e0722eea69e94b4e01b387386252dacb76200c512d2390d1579647d6863c8138f5e2ee43d3bbb1cd9112552

data/CHANGELOG.md

@@ -1,5 +1,10 @@
+## 4.0.0
+  - breaking: trim and trimkey options are renamed to trim_value and trim_key
+  - bugfix: trim_value and trim_key options now remove only leading and trailing characters (#10)
+  - feature: new options remove_char_value and remove_char_key to remove all characters from keys/values whatever their position
+
 ## 3.1.1
-  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
+  - internal,deps: Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 
 ## 3.1.0
   - Adds :transform_value and :transform_key options to lowercase/upcase or capitalize all keys/values

data/CONTRIBUTORS

@@ -19,6 +19,7 @@ Contributors:
 * Scott Bessler (scottbessler)
 * Suyog Rao (suyograo)
 * piavlo
+* Fabien Baligand (fbaligand)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/filters/kv.rb

@@ -41,14 +41,16 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # These characters form a regex character class and thus you must escape special regex
   # characters like `[` or `]` using `\`.
   #
-  # For example, to strip `<`, `>`, `[`, `]` and `,` characters from values:
+  # Only leading and trailing characters are trimed from the value.
+  #
+  # For example, to trim `<`, `>`, `[`, `]` and `,` characters from values:
   # [source,ruby]
   #     filter {
   #       kv {
-  #         trim => "<>\[\],"
+  #         trim_value => "<>\[\],"
   #       }
   #     }
-  config :trim, :validate => :string
+  config :trim_value, :validate => :string
 
   # A string of characters to trim from the key. This is useful if your
   # keys are wrapped in brackets or start with space.
@@ -56,14 +58,48 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # These characters form a regex character class and thus you must escape special regex
   # characters like `[` or `]` using `\`.
   #
-  # For example, to strip `<` `>` `[` `]` and `,` characters from keys:
+  # Only leading and trailing characters are trimed from the key.
+  #
+  # For example, to trim `<` `>` `[` `]` and `,` characters from keys:
+  # [source,ruby]
+  #     filter {
+  #       kv {
+  #         trim_key => "<>\[\],"
+  #       }
+  #     }
+  config :trim_key, :validate => :string
+
+  # A string of characters to remove from the value.
+  #
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like `[` or `]` using `\`.
+  #
+  # Contrary to trim option, all characters are removed from the value, whatever their position.
+  #
+  # For example, to remove `<`, `>`, `[`, `]` and `,` characters from values:
   # [source,ruby]
   #     filter {
   #       kv {
-  #         trimkey => "<>\[\],"
+  #         remove_char_value => "<>\[\],"
   #       }
   #     }
-  config :trimkey, :validate => :string
+  config :remove_char_value, :validate => :string
+
+  # A string of characters to remove from the key.
+  #
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like `[` or `]` using `\`.
+  #
+  # Contrary to trim option, all characters are removed from the key, whatever their position.
+  #
+  # For example, to remove `<` `>` `[` `]` and `,` characters from keys:
+  # [source,ruby]
+  #     filter {
+  #       kv {
+  #         remove_char_key => "<>\[\],"
+  #       }
+  #     }
+  config :remove_char_key, :validate => :string
 
   # Transform values to lower case, upper case or capitals.
   #
@@ -190,7 +226,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # one unique key/value pair will be preserved.
   #
   # For example, consider a source like `from=me from=me`. `[from]` will map to
-  # an Array with two elements: `["me", "me"]`. to only keep unique key/value pairs,
+  # an Array with two elements: `["me", "me"]`. To only keep unique key/value pairs,
   # you could use this configuration:
   # [source,ruby]
   #     filter {
@@ -250,8 +286,11 @@ class LogStash::Filters::KV < LogStash::Filters::Base
       )
     end
 
-    @trim_re = Regexp.new("[#{@trim}]") if @trim
-    @trimkey_re = Regexp.new("[#{@trimkey}]") if @trimkey
+    @trim_value_re = Regexp.new("^[#{@trim_value}]|[#{@trim_value}]$") if @trim_value
+    @trim_key_re = Regexp.new("^[#{@trim_key}]|[#{@trim_key}]$") if @trim_key
+
+    @remove_char_value_re = Regexp.new("[#{@remove_char_value}]") if @remove_char_value
+    @remove_char_key_re = Regexp.new("[#{@remove_char_key}]") if @remove_char_key
 
     valueRxString = "(?:\"([^\"]+)\"|'([^']+)'"
     valueRxString += "|\\(([^\\)]+)\\)|\\[([^\\]]+)\\]|<([^>]+)>" if @include_brackets
@@ -317,7 +356,8 @@ class LogStash::Filters::KV < LogStash::Filters::Base
 
     text.scan(@scan_re) do |key, v1, v2, v3, v4, v5, v6|
       value = v1 || v2 || v3 || v4 || v5 || v6
-      key = @trimkey ? key.gsub(@trimkey_re, "") : key
+      key = @trim_key ? key.gsub(@trim_key_re, "") : key
+      key = @remove_char_key ? key.gsub(@remove_char_key_re, "") : key
       key = @transform_key ? transform(key, @transform_key) : key
 
       # Bail out as per the values of include_keys and exclude_keys
@@ -327,7 +367,8 @@ class LogStash::Filters::KV < LogStash::Filters::Base
 
       key = event.sprintf(@prefix) + key
 
-      value = @trim ? value.gsub(@trim_re, "") : value
+      value = @trim_value ? value.gsub(@trim_value_re, "") : value
+      value = @remove_char_value ? value.gsub(@remove_char_value_re, "") : value
       value = @transform_value ? transform(value, @transform_value) : value
 
       # Bail out if inserting duplicate value in key mapping when unique_values

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '3.1.1'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter helps automatically parse messages (or specific event fields) which are of the 'foo=bar' variety."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/kv_spec.rb

@@ -676,7 +676,7 @@ describe LogStash::Filters::KV do
       {
         "field_split" => "\r\n",
         "value_split" => " ",
-        "trimkey" => ":"
+        "trim_key" => ":"
       }
     }
 
@@ -686,7 +686,67 @@ describe LogStash::Filters::KV do
         expect(event.get("AccountStatus")).to eq("4")
         expect(event.get("IsSuccess")).to eq("True")
         expect(event.to_hash.keys.sort).to eq(
-          ["@timestamp", "@version", "AccountStatus", "IsSuccess", "message"])
+          ["@timestamp", "@version", "AccountStatus", "IsSuccess", "message", "tags"])
+      end
+    end
+  end
+
+  describe "trim_key/trim_value options : trim only leading and trailing spaces in keys/values (reported in #10)" do
+    subject do
+      plugin = LogStash::Filters::KV.new(options)
+      plugin.register
+      plugin
+    end
+
+    let(:message) { "key1= value1 with spaces | key2 with spaces =value2" }
+    let(:data) { {"message" => message} }
+    let(:event) { LogStash::Event.new(data) }
+    let(:options) {
+      {
+        "field_split" => "\|",
+        "value_split" => "=",
+        "trim_value" => " ",
+        "trim_key" => " "
+      }
+    }
+
+    context "key and value with leading, trailing and middle spaces" do
+      it "should trim only leading and trailing spaces" do
+        subject.filter(event)
+        expect(event.get("key1")).to eq("value1 with spaces")
+        expect(event.get("key2 with spaces")).to eq("value2")
+        expect(event.to_hash.keys.sort).to eq(
+          ["@timestamp", "@version", "key1", "key2 with spaces", "message", "tags"])
+      end
+    end
+  end
+
+  describe "remove_char_key/remove_char_value options : remove all characters in keys/values whatever their position" do
+    subject do
+      plugin = LogStash::Filters::KV.new(options)
+      plugin.register
+      plugin
+    end
+  
+    let(:message) { "key1= value1 with spaces | key2 with spaces =value2" }
+    let(:data) { {"message" => message} }
+    let(:event) { LogStash::Event.new(data) }
+    let(:options) {
+      {
+        "field_split" => "\|",
+        "value_split" => "=",
+        "remove_char_value" => " ",
+        "remove_char_key" => " "
+      }
+    }
+  
+    context "key and value with leading, trailing and middle spaces" do
+      it "should remove all spaces" do
+        subject.filter(event)
+        expect(event.get("key1")).to eq("value1withspaces")
+        expect(event.get("key2withspaces")).to eq("value2")
+        expect(event.to_hash.keys.sort).to eq(
+          ["@timestamp", "@version", "key1", "key2withspaces", "message", "tags"])
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2017-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -81,7 +81,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This filter helps automatically parse messages (or specific event fields) which are of the 'foo=bar' variety.