logstash-filter-kv 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 67fc628e91472f712e8195aaff0bea1cce4e1538
-  data.tar.gz: 84912044f62dc43af4529cac310638b72ca8a26f
+  metadata.gz: 7823f827ecc2a7359e187f76eeef30ace026483d
+  data.tar.gz: 15efba89f825cfe3431ad6fc66f1fa8eb06ff1d9
 SHA512:
-  metadata.gz: 3c4deb2402e12be72d4660fa567e01d29802e1024b0571dd7c3ce42f08237e0c7dcbc85b02cc2c9c68dc64070be2e6bb93ab5839b52114410ac746abd9bf6a03
-  data.tar.gz: 7626be289be507fa32ebee6669d39f29805be40b5d1947d052da14837652d2c1b54be5f5fe5fe93784362418a4d554e569207281709fe4896efe935019b5f450
+  metadata.gz: b5c3322e32a0c28e133ec2a7b7a347019e208c70d7ec84afca3ec086c1d0389f3b20effce8392a8646b333e378dfa4a917d00f7b92731da3c2953ce6291d8604
+  data.tar.gz: f0fd946fba942d8e8c5d0c678e85619b522b32e12ba6b2bce49e2bf6166cc6744b5ed9838e94ebc267565b1336304778b8551d431e01978d3b11336b041d63a4

data/CHANGELOG.md

@@ -1,5 +1,9 @@
+## 2.0.3
+ - fixed fixed short circuit expressions, some optimizations, added specs, PR #20
+ - fixed event field assignment, PR #21
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-kv-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-kv-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/filters/kv.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+
 require "logstash/filters/base"
 require "logstash/namespace"
 
@@ -124,7 +125,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # An array specifying the parsed keys which should be added to the event.
   # By default all keys will be added.
   #
-  # For example, consider a source like `Hey, from=<abc>, to=def foo=bar`. 
+  # For example, consider a source like `Hey, from=<abc>, to=def foo=bar`.
   # To include `from` and `to`, but exclude the `foo` key, you could use this configuration:
   # [source,ruby]
   #     filter {
@@ -137,7 +138,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # An array specifying the parsed keys which should not be added to the event.
   # By default no keys will be excluded.
   #
-  # For example, consider a source like `Hey, from=<abc>, to=def foo=bar`. 
+  # For example, consider a source like `Hey, from=<abc>, to=def foo=bar`.
   # To exclude `from` and `to`, but retain the `foo` key, you could use this configuration:
   # [source,ruby]
   #     filter {
@@ -158,10 +159,10 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #     }
   config :default_keys, :validate => :hash, :default => {}
 
-  # A bool option for removing duplicate key/value pairs. When set to false, only 
+  # A bool option for removing duplicate key/value pairs. When set to false, only
   # one unique key/value pair will be preserved.
   #
-  # For example, consider a source like `from=me from=me`. `[from]` will map to 
+  # For example, consider a source like `from=me from=me`. `[from]` will map to
   # an Array with two elements: `["me", "me"]`. to only keep unique key/value pairs,
   # you could use this configuration:
   # [source,ruby]
@@ -193,7 +194,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # * bracketstwo: [hello
   config :include_brackets, :validate => :boolean, :default => true
 
-  # A boolean specifying whether to drill down into values 
+  # A boolean specifying whether to drill down into values
   # and recursively get more key-value pairs from it.
   # The extra key-value pairs will be stored as subkeys of the root key.
   #
@@ -208,90 +209,86 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   config :recursive, :validate => :boolean, :default => false
 
   def register
-    @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil?
-    @trimkey_re = Regexp.new("[#{@trimkey}]") if !@trimkey.nil?
+    @trim_re = Regexp.new("[#{@trim}]") if @trim
+    @trimkey_re = Regexp.new("[#{@trimkey}]") if @trimkey
 
     valueRxString = "(?:\"([^\"]+)\"|'([^']+)'"
     valueRxString += "|\\(([^\\)]+)\\)|\\[([^\\]]+)\\]" if @include_brackets
-    valueRxString += "|((?:\\\\ |[^"+@field_split+"])+))"
-    @scan_re = Regexp.new("((?:\\\\ |[^"+@field_split+@value_split+"])+)\\s*["+@value_split+"]\\s*"+valueRxString)
-  end # def register
+    valueRxString += "|((?:\\\\ |[^" + @field_split + "])+))"
+    @scan_re = Regexp.new("((?:\\\\ |[^" + @field_split + @value_split + "])+)\\s*[" + @value_split + "]\\s*" + valueRxString)
 
-  def filter(event)
-    
+    @value_split_re = /[#{@value_split}]/
+  end
 
+  def filter(event)
     kv = Hash.new
-
     value = event[@source]
 
     case value
-      when nil; # Nothing to do
-      when String; kv = parse(value, event, kv)
-      when Array; value.each { |v| kv = parse(v, event, kv) }
-      else
-        @logger.warn("kv filter has no support for this type of data",
-                     :type => value.class, :value => value)
-    end # case value
+    when nil
+      # Nothing to do
+    when String
+      kv = parse(value, event, kv)
+    when Array
+      value.each { |v| kv = parse(v, event, kv) }
+    else
+      @logger.warn("kv filter has no support for this type of data", :type => value.class, :value => value)
+    end
 
     # Add default key-values for missing keys
     kv = @default_keys.merge(kv)
 
-    # If we have any keys, create/append the hash
-    if kv.length > 0
-      if @target.nil?
-        # Default is to write to the root of the event.
-        dest = event.to_hash
-      else
-        if !event[@target].is_a?(Hash)
-          @logger.debug("Overwriting existing target field", :target => @target)
-          dest = event[@target] = {}
-        else
-          dest = event[@target]
-        end
-      end
+    return if kv.empty?
 
-      dest.merge!(kv)
-      filter_matched(event)
+    if @target
+      @logger.debug? && @logger.debug("Overwriting existing target field", :target => @target)
+      event[@target] = kv
+    else
+      kv.each{|k, v| event[k] = v}
     end
-  end # def filter
+
+    filter_matched(event)
+  end
 
   private
+
+  def has_value_splitter?(s)
+    s =~ @value_split_re
+  end
+
   def parse(text, event, kv_keys)
-    if !event =~ /[@value_split]/
-      return kv_keys
-    end
-    
+    # short circuit parsing if the text does not contain the @value_split
+    return kv_keys unless has_value_splitter?(text)
+
     # Interpret dynamic keys for @include_keys and @exclude_keys
     include_keys = @include_keys.map{|key| event.sprintf(key)}
     exclude_keys = @exclude_keys.map{|key| event.sprintf(key)}
-    
+
     text.scan(@scan_re) do |key, v1, v2, v3, v4, v5|
       value = v1 || v2 || v3 || v4 || v5
-      key = @trimkey.nil? ? key : key.gsub(@trimkey_re, "")
-      
+      key = @trimkey ? key.gsub(@trimkey_re, "") : key
+
       # Bail out as per the values of include_keys and exclude_keys
       next if not include_keys.empty? and not include_keys.include?(key)
+      # next unless include_keys.include?(key)
       next if exclude_keys.include?(key)
 
       key = event.sprintf(@prefix) + key
 
-      value = @trim.nil? ? value : value.gsub(@trim_re, "")
+      value = @trim ? value.gsub(@trim_re, "") : value
 
-      # Bail out if inserting duplicate value in key mapping when unique_values 
+      # Bail out if inserting duplicate value in key mapping when unique_values
       # option is set to true.
       next if not @allow_duplicate_values and kv_keys.has_key?(key) and kv_keys[key].include?(value)
 
       # recursively get more kv pairs from the value
-      if @recursive and value =~ /[@value_split]/
-        innerKv = Hash.new
-        innerKv = parse(value, event, innerKv) 
-        if innerKv.length > 0
-          value = innerKv
-        end
+      if @recursive
+        innerKv = parse(value, event, {})
+        value = innerKv unless innerKv.empty?
       end
 
       if kv_keys.has_key?(key)
-        if kv_keys[key].is_a? Array
+        if kv_keys[key].is_a?(Array)
           kv_keys[key].push(value)
         else
           kv_keys[key] = [kv_keys[key], value]
@@ -300,6 +297,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
         kv_keys[key] = value
       end
     end
+
     return kv_keys
   end
-end # class LogStash::Filters::KV
+end

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter helps automatically parse messages (or specific event fields) which are of the 'foo=bar' variety."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/kv_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/kv"
 
@@ -54,21 +56,108 @@ describe LogStash::Filters::KV do
   end
 
   describe "test value_split" do
-    config <<-CONFIG
-      filter {
-        kv { value_split => ':' }
-      }
-    CONFIG
+    context "using an alternate splitter" do
+      config <<-CONFIG
+        filter {
+          kv { value_split => ':' }
+        }
+      CONFIG
 
-    sample "hello:=world foo:bar baz=:fizz doublequoted:\"hello world\" singlequoted:'hello world' brackets:(hello world)" do
-      insist { subject["hello"] } == "=world"
-      insist { subject["foo"] } == "bar"
-      insist { subject["baz="] } == "fizz"
-      insist { subject["doublequoted"] } == "hello world"
-      insist { subject["singlequoted"] } == "hello world"
-      insist { subject["brackets"] } == "hello world"
+      sample "hello:=world foo:bar baz=:fizz doublequoted:\"hello world\" singlequoted:'hello world' brackets:(hello world)" do
+        insist { subject["hello"] } == "=world"
+        insist { subject["foo"] } == "bar"
+        insist { subject["baz="] } == "fizz"
+        insist { subject["doublequoted"] } == "hello world"
+        insist { subject["singlequoted"] } == "hello world"
+        insist { subject["brackets"] } == "hello world"
+      end
+    end
+  end
+
+  # these specs are quite implementation specific by testing on the private method
+  # has_value_splitter?  - this is what I figured would help fixing the short circuit
+  # broken code that was previously in place
+  describe "short circuit" do
+    subject do
+      plugin = LogStash::Filters::KV.new(options)
+      plugin.register
+      plugin
+    end
+    let(:data) { {"message" => message} }
+    let(:event) { LogStash::Event.new(data) }
+
+    context "plain message" do
+      let(:options) { {} }
+
+      context "without splitter" do
+        let(:message) { "foo:bar" }
+        it "should short circuit" do
+          expect(subject.send(:has_value_splitter?, message)).to be_falsey
+          expect(subject).to receive(:has_value_splitter?).with(message).once.and_return(false)
+          subject.filter(event)
+        end
+      end
+
+      context "with splitter" do
+        let(:message) { "foo=bar" }
+        it "should not short circuit" do
+          expect(subject.send(:has_value_splitter?, message)).to be_truthy
+          expect(subject).to receive(:has_value_splitter?).with(message).once.and_return(true)
+          subject.filter(event)
+        end
+      end
     end
 
+    context "recursive message" do
+      context "without inner splitter" do
+        let(:inner) { "bar" }
+        let(:message) { "foo=#{inner}" }
+        let(:options) { {"recursive" => "true"} }
+
+        it "should extract kv" do
+          subject.filter(event)
+          expect(event["foo"]).to eq(inner)
+        end
+
+        it "should short circuit" do
+          expect(subject.send(:has_value_splitter?, message)).to be_truthy
+          expect(subject.send(:has_value_splitter?, inner)).to be_falsey
+          expect(subject).to receive(:has_value_splitter?).with(message).once.and_return(true)
+          expect(subject).to receive(:has_value_splitter?).with(inner).once.and_return(false)
+          subject.filter(event)
+        end
+      end
+
+      context "with inner splitter" do
+        let(:foo_val) { "1" }
+        let(:baz_val) { "2" }
+        let(:inner) { "baz=#{baz_val}" }
+        let(:message) { "foo=#{foo_val} bar=(#{inner})" } # foo=1 bar=(baz=2)
+        let(:options) { {"recursive" => "true"} }
+
+        it "should extract kv" do
+          subject.filter(event)
+          expect(event["foo"]).to eq(foo_val)
+          expect(event["[bar][baz]"]).to eq(baz_val)
+        end
+
+        it "should short circuit" do
+          expect(subject.send(:has_value_splitter?, message)).to be_truthy
+          expect(subject.send(:has_value_splitter?, foo_val)).to be_falsey
+
+          expect(subject.send(:has_value_splitter?, inner)).to be_truthy
+          expect(subject.send(:has_value_splitter?, baz_val)).to be_falsey
+
+          expect(subject).to receive(:has_value_splitter?).with(message).once.and_return(true)
+          expect(subject).to receive(:has_value_splitter?).with(foo_val).once.and_return(false)
+
+          expect(subject).to receive(:has_value_splitter?).with(inner).once.and_return(true)
+          expect(subject).to receive(:has_value_splitter?).with(baz_val).once.and_return(false)
+
+          subject.filter(event)
+        end
+      end
+    end
   end
 
   describe "test field_split" do
@@ -104,12 +193,12 @@ describe LogStash::Filters::KV do
   describe "test recursive" do
     config <<-CONFIG
       filter {
-        kv { 
+        kv {
           recursive => 'true'
         }
       }
     CONFIG
-  
+
     sample 'IKE="Quick Mode completion" IKE\ IDs = (subnet= x.x.x.x mask= 255.255.255.254 and host=y.y.y.y)' do
       insist { subject["IKE"] } == 'Quick Mode completion'
       insist { subject['IKE\ IDs']['subnet'] } == 'x.x.x.x'
@@ -393,7 +482,7 @@ describe LogStash::Filters::KV do
       insist { subject["__doublequoted"] } == "hello world"
     end
   end
-  
+
   describe "test include_keys with dynamic key" do
     config <<-CONFIG
       filter {
@@ -403,13 +492,13 @@ describe LogStash::Filters::KV do
         }
       }
     CONFIG
-    
+
     sample({"data" => "foo=bar baz=fizz", "key" => "foo"}) do
       insist { subject["foo"] } == "bar"
       insist { subject["baz"] } == nil
     end
   end
-  
+
   describe "test exclude_keys with dynamic key" do
     config <<-CONFIG
       filter {
@@ -419,7 +508,7 @@ describe LogStash::Filters::KV do
         }
       }
     CONFIG
-    
+
     sample({"data" => "foo=bar baz=fizz", "key" => "foo"}) do
       insist { subject["foo"] } == nil
       insist { subject["baz"] } == "fizz"

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,20 +28,22 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []