logstash-filter-kv 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0453238168d043ddc73127a20d36983decd681e4
-  data.tar.gz: 37067199c62c04fbd097a5b0c75257348af0950b
+  metadata.gz: 8d91c83bc99167f5507bedad6cd5b45979c56ad1
+  data.tar.gz: 46c64b9469e5a4e12e5f96a6e39840fdb6c253f8
 SHA512:
-  metadata.gz: 4f6ac707d371f8bdc310adabfd197d0c2266e0817ed7a4cd8d8079e1ee7c32bfc7a8222db81f3cc271911ddaf66f0dfe780c0042065c25e8ab516878a5c5a357
-  data.tar.gz: b0c36b4d5f6fbc3191c9ae6abc48fa2d0c3190240ca72e863712d01e438a733ae5091d32bd959d17bb9619ed7e29ac7bf8a042ffdc12964ebadb6a6bd0a5bd64
+  metadata.gz: 49d79377ad19c67357c5739a249d21dc4d5253340dc75cb0241702d5f91f4232347ec894732910dc5cbe6ab1f49b2b88e3903c10d5e11747559fbbc965061d8a
+  data.tar.gz: 855dc47570be28ece7aecec466d80aece0ca744e5fc4ec9155095f1fbc57a41195e145ca7ac34b82545ddb722358962762288ee478a73b705fd6ff9de9638ead

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.1.0
+ - Add new features as support spaces between key and value_split,
+   support brackets and recursive option.

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/kv.rb

@@ -172,10 +172,49 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #     }
   config :allow_duplicate_values, :validate => :boolean, :default => true
 
+  # A boolean specifying whether to include brackets as value `wrappers`
+  # (the default is true)
+  # [source,ruby]
+  #     filter {
+  #       kv {
+  #         include_brackets => true
+  #       }
+  #     }
+  #
+  # For example, the result of this line:
+  # `bracketsone=(hello world) bracketstwo=[hello world]`
+  #
+  # will be:
+  # * bracketsone: hello world
+  # * bracketstwo: hello world
+  #
+  # instead of:
+  # * bracketsone: (hello
+  # * bracketstwo: [hello
+  config :include_brackets, :validate => :boolean, :default => true
+
+  # A boolean specifying whether to drill down into values 
+  # and recursively get more key-value pairs from it.
+  # The extra key-value pairs will be stored as subkeys of the root key.
+  #
+  # Default is not to recursive values.
+  # [source,ruby]
+  #     filter {
+  #       kv {
+  #         recursive => "true"
+  #       }
+  #     }
+  #
+  config :recursive, :validate => :boolean, :default => false
+
   def register
     @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil?
     @trimkey_re = Regexp.new("[#{@trimkey}]") if !@trimkey.nil?
-    @scan_re = Regexp.new("((?:\\\\ |[^"+@field_split+@value_split+"])+)["+@value_split+"](?:\"([^\"]+)\"|'([^']+)'|((?:\\\\ |[^"+@field_split+"])+))")
+
+    valueRxString = "(?:\"([^\"]+)\"|'([^']+)'"
+    valueRxString += "|\\(([^\\)]+)\\)|\\[([^\\]]+)\\]" if @include_brackets
+    valueRxString += "|((?:\\\\ |[^"+@field_split+"])+))"
+    @scan_re = Regexp.new("((?:\\\\ |[^"+@field_split+@value_split+"])+)\\s*["+@value_split+"]\\s*"+valueRxString)
   end # def register
 
   def filter(event)
@@ -218,7 +257,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
 
   private
   def parse(text, event, kv_keys)
-    if !event =~ /[@field_split]/
+    if !event =~ /[@value_split]/
       return kv_keys
     end
     
@@ -226,8 +265,8 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     include_keys = @include_keys.map{|key| event.sprintf(key)}
     exclude_keys = @exclude_keys.map{|key| event.sprintf(key)}
     
-    text.scan(@scan_re) do |key, v1, v2, v3|
-      value = v1 || v2 || v3
+    text.scan(@scan_re) do |key, v1, v2, v3, v4, v5|
+      value = v1 || v2 || v3 || v4 || v5
       key = @trimkey.nil? ? key : key.gsub(@trimkey_re, "")
       
       # Bail out as per the values of include_keys and exclude_keys
@@ -242,6 +281,15 @@ class LogStash::Filters::KV < LogStash::Filters::Base
       # option is set to true.
       next if not @allow_duplicate_values and kv_keys.has_key?(key) and kv_keys[key].include?(value)
 
+      # recursively get more kv pairs from the value
+      if @recursive and value =~ /[@value_split]/
+        innerKv = Hash.new
+        innerKv = parse(value, event, innerKv) 
+        if innerKv.length > 0
+          value = innerKv
+        end
+      end
+
       if kv_keys.has_key?(key)
         if kv_keys[key].is_a? Array
           kv_keys[key].push(value)

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '1.0.0'
+  s.version         = '2.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter helps automatically parse messages (or specific event fields) which are of the 'foo=bar' variety."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/kv_spec.rb

@@ -12,14 +12,32 @@ describe LogStash::Filters::KV do
       }
     CONFIG
 
-    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world' bracketsone=(hello world) bracketstwo=[hello world]" do
       insist { subject["hello"] } == "world"
       insist { subject["foo"] } == "bar"
       insist { subject["baz"] } == "fizz"
       insist { subject["doublequoted"] } == "hello world"
       insist { subject["singlequoted"] } == "hello world"
+      insist { subject["bracketsone"] } == "hello world"
+      insist { subject["bracketstwo"] } == "hello world"
     end
+  end
+
+  describe  "test spaces attached to the field_split" do
+    config <<-CONFIG
+      filter {
+        kv { }
+      }
+    CONFIG
 
+    sample "hello = world foo =bar baz= fizz doublequoted = \"hello world\" singlequoted= 'hello world' brackets =(hello world)" do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+      insist { subject["brackets"] } == "hello world"
+    end
   end
 
    describe "LOGSTASH-624: allow escaped space in key or value " do
@@ -42,12 +60,13 @@ describe LogStash::Filters::KV do
       }
     CONFIG
 
-    sample "hello:=world foo:bar baz=:fizz doublequoted:\"hello world\" singlequoted:'hello world'" do
+    sample "hello:=world foo:bar baz=:fizz doublequoted:\"hello world\" singlequoted:'hello world' brackets:(hello world)" do
       insist { subject["hello"] } == "=world"
       insist { subject["foo"] } == "bar"
       insist { subject["baz="] } == "fizz"
       insist { subject["doublequoted"] } == "hello world"
       insist { subject["singlequoted"] } == "hello world"
+      insist { subject["brackets"] } == "hello world"
     end
 
   end
@@ -67,7 +86,36 @@ describe LogStash::Filters::KV do
       insist { subject["singlequoted"] } == "hello world"
       insist { subject["foo12"] } == "bar12"
     end
+  end
+
+  describe "test include_brackets is false" do
+    config <<-CONFIG
+      filter {
+        kv { include_brackets => "false" }
+      }
+    CONFIG
+
+    sample "bracketsone=(hello world) bracketstwo=[hello world]" do
+      insist { subject["bracketsone"] } == "(hello"
+      insist { subject["bracketstwo"] } == "[hello"
+    end
+  end
 
+  describe "test recursive" do
+    config <<-CONFIG
+      filter {
+        kv { 
+          recursive => 'true'
+        }
+      }
+    CONFIG
+  
+    sample 'IKE="Quick Mode completion" IKE\ IDs = (subnet= x.x.x.x mask= 255.255.255.254 and host=y.y.y.y)' do
+      insist { subject["IKE"] } == 'Quick Mode completion'
+      insist { subject['IKE\ IDs']['subnet'] } == 'x.x.x.x'
+      insist { subject['IKE\ IDs']['mask'] } == '255.255.255.254'
+      insist { subject['IKE\ IDs']['host'] } == 'y.y.y.y'
+    end
   end
 
   describe  "delimited fields should override space default (reported by LOGSTASH-733)" do

metadata

@@ -1,63 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.0.snapshot
+  name: logstash-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0.snapshot
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
-- Rakefile
 - lib/logstash/filters/kv.rb
 - logstash-filter-kv.gemspec
 - spec/filters/kv_spec.rb
@@ -83,7 +75,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This filter helps automatically parse messages (or specific event fields) which are of the 'foo=bar' variety.

data/.gitignore

@@ -1,4 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle
-vendor

data/Rakefile

@@ -1,7 +0,0 @@
-@files=[]
-
-task :default do
-  system("rake -T")
-end
-
-require "logstash/devutils/rake"