logstash-filter-kv 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a2f193234be65735a775b9c6f760fdb39b0e147e
-  data.tar.gz: 51155886466309a0c92d42778617e8f038efb4bb
+  metadata.gz: 5a4015b34b2918460e579fb2bae5e8a9bcc61420
+  data.tar.gz: 97bb73a8e6f988a384753a69ade43fc13b390fc7
 SHA512:
-  metadata.gz: 30c7992eb2dcd323d6fbfa26de7f92a7615c0db51200870295d9b5279b4829d760a6343b292898009175e2c95f0dfb671050b6756406f25593cec12a80e66993
-  data.tar.gz: c6abe01f5f9d4bf460b42752f11b8c71aa4e11162ffba5190919686efa41fbcca11bfdf4413c6f8841f5b71b4ea2cdf2f4fe8dbbf1f6d21da2352c8821c5df21
+  metadata.gz: 881f352227e3d6bc86812f8d04152761f45f4ecbba7b2e9829b01db87b2f97791b2204c7fdab06e5d41aab006d38f8d3fbb72cfc50b5f26a39a7ce91c8fae07f
+  data.tar.gz: 2824c28b60c78ca4cda3eee1514d0804edc1f1544cf4e87522ae32320a37c4cfe923c0dc0c4a6ca964f8f72e21fd3b6652329fc613d4742361d8de22f65c6614

data/lib/logstash/filters/kv.rb

@@ -158,6 +158,20 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #     }
   config :default_keys, :validate => :hash, :default => {}
 
+  # A bool option for removing duplicate key/value pairs. When set to false, only 
+  # one unique key/value pair will be preserved.
+  #
+  # For example, consider a source like `from=me from=me`. `[from]` will map to 
+  # an Array with two elements: `["me", "me"]`. to only keep unique key/value pairs,
+  # you could use this configuration:
+  # [source,ruby]
+  #     filter {
+  #       kv {
+  #         allow_duplicate_values => false
+  #       }
+  #     }
+  config :allow_duplicate_values, :validate => :boolean, :default => true
+
   def register
     @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil?
     @trimkey_re = Regexp.new("[#{@trimkey}]") if !@trimkey.nil?
@@ -223,6 +237,11 @@ class LogStash::Filters::KV < LogStash::Filters::Base
       key = event.sprintf(@prefix) + key
 
       value = @trim.nil? ? value : value.gsub(@trim_re, "")
+
+      # Bail out if inserting duplicate value in key mapping when unique_values 
+      # option is set to true.
+      next if not @allow_duplicate_values and kv_keys.has_key?(key) and kv_keys[key].include?(value)
+
       if kv_keys.has_key?(key)
         if kv_keys[key].is_a? Array
           kv_keys[key].push(value)

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '0.1.3'
+  s.version         = '0.1.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter helps automatically parse messages (or specific event fields) which are of the 'foo=bar' variety."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/kv_spec.rb

@@ -433,4 +433,34 @@ describe LogStash::Filters::KV do
 
   end
 
+  describe "Removing duplicate key/value pairs" do
+    config <<-CONFIG
+      filter {
+        kv {
+          field_split => "&"
+          source => "source"
+          allow_duplicate_values => false
+        }
+      }
+    CONFIG
+
+    sample("source" => "foo=bar&foo=yeah&foo=yeah") do
+      insist { subject["[foo]"] } == ["bar", "yeah"]
+    end
+  end
+
+  describe "Allow duplicate key/value pairs by default" do
+    config <<-CONFIG
+      filter {
+        kv {
+          field_split => "&"
+          source => "source"
+        }
+      }
+    CONFIG
+
+    sample("source" => "foo=bar&foo=yeah&foo=yeah") do
+      insist { subject["[foo]"] } == ["bar", "yeah", "yeah"]
+    end
+  end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-27 00:00:00.000000000 Z
+date: 2015-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,20 +28,22 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elasticsearch.com
 executables: []