logstash-filter-list2fields 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9832818142f818906d8759e43dbc910d12f5e37f
-  data.tar.gz: 63aa2f896df825be60ffe05a4e2a6ed04c95c66d
+  metadata.gz: 484bb95370a4baf2895cda0b34310b2c698192cc
+  data.tar.gz: ba91bfb3dca14654a53c35b8c45648f52e750cd9
 SHA512:
-  metadata.gz: 7d4417e96eec33160430d59e4eb4b54dc7b9aaa3cef9ef6aa59983410538ba59814ca54760c9a26d144898753fc1ceb7b9f29c4d61c34519fe408a4b66eee4ce
-  data.tar.gz: 17171e3a3477456cde677cc483979983b074ebfd8bdeab622d7889f310b0b7e175a8c131e9f0f34842dc521e59265054a78b45b3caed0ef2740ed8c85d863442
+  metadata.gz: b2ebdeefc163d42dcdd9411603a84a863a636378e8c213acd1a8c585ee4f872376bd7317608ab57b73e3333ae61773ef8f8e8d05e21e508814304b14ae12fe64
+  data.tar.gz: 3bb28663abced9e74f534d57112a940f2af02c121f2f9797e4f895a6f6d480ab17e86e50ce25a0622e75492de6503f476480000cc630025820c588aceaf3058e

data/README.md

@@ -1,15 +1,25 @@
-# Logstash Plugin
+# Logstash list2fields Filter Plugin
 
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
+## Installation
+
+You can download the plugin from [rubygems](https://rubygems.org/gems/logstash-filter-list2fields) and install it from your logstash home directory like so:
+
+	bin/plugin install logstash-filter-list2fields-$VERSION.gem
+
 ## Documentation
 
-This filter rearranges key-value-pairs from the following structure
+This filter rearranges key-value-pairs from the following structures
+
+	example 1: 			"awesome_list" => [ { "animal" => "horse"} , {"food" => "bacon"} ]
+
+or
   
-	"awesome_list" => [ { "key" => "animal", "value" => "horse"} , {"key" => "food", "value" => "bacon"} ]
+	example 2: 			"awesome_list" => [ { "key" => "animal", "value" => "horse"} , {"key" => "food", "value" => "bacon"} ]
 
 into separate logstash event fields:
 
@@ -19,11 +29,11 @@ into separate logstash event fields:
 ## Configuration
 
 * source 	: name of the field which contains the list of key value pairs (required). In the example above, this would be "awesome_list".
-* key 		: name of the key for the key field in the list of key value pairs. Headache? Example: If this is your incoming data:
-	"awesome_list" => [ { "name" => "animal", "content" => "horse"} , {"name" => "food", "content" => "bacon"} ]
-then the key would be "name". Optional, defaults to "key".
-* value 	: name of the key for the value field in the list of key value pairs. In the above example, this would be "content". Optional, defaults to "value".
-* prefix	: string to prepend to the new fields that will be added to the logstash event. Optional, defaults to empty.
+* prefix	: string to prepend to the new fields that will be added to the logstash event. Optional.
+* key 		: optional for example 1, required for example #2: name of the key for the key field in the list of key value pairs. Headache? Example: If this is your incoming data:
+	example 3: 			"awesome_list" => [ { "name" => "animal", "content" => "horse"} , {"name" => "food", "content" => "bacon"} ]
+then the key would be "name".
+* value 	: optional for example 1, required for example #2: name of the key for the value field in the list of key value pairs. In example 3, this would be "content".
 
 
 ## Contributing

data/lib/logstash/filters/list2fields.rb

@@ -2,7 +2,6 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 
-# TODO docu
 class LogStash::Filters::List2fields < LogStash::Filters::Base
   
   config_name "list2fields"
@@ -10,37 +9,69 @@ class LogStash::Filters::List2fields < LogStash::Filters::Base
   # The name of the field which contains the list of key value pairs
   config :source, :validate => :string
 
-  # The name of the field which contains the key inside a list element
-  config :key, :validate => :string, :default => "key"
+  # The name of the field which contains the key inside a list element. If this is set then "value" needs to be set also.
+  config :key, :validate => :string, :default => ""
 
-  # The name of the field which contains the value inside a list element
-  config :value, :validate => :string, :default => "value"
+  # The name of the field which contains the value inside a list element. If this is set then "key" needs to be set also.
+  config :value, :validate => :string, :default => ""
  
   # Prefix for the elements that will be added.
   config :prefix, :validate => :string, :default => ""
 
+  # Remove source field after transformation
+  config :remove_source, :validate => :boolean, :default => true
+
+  # if you want to have debug information for this plugin only and you dont want to set the whole logstash to debug.
+  config :debug, :validate => :boolean, :default => false
+
   public
   def register
-    
+    @access_by_name = !@key.empty? && !@value.empty?   
   end # def register
 
   public
   def filter(event)
-    input = event[@source]
-    if !input.nil?  && (input.is_a? Enumerable)
+    input = event.get(@source)
+    if !input.nil?  && (input.is_a? Enumerable)     
       input.each do |entry|
         begin
-          if entry.is_a?(::Hash)
-            new_key = @prefix.to_s + entry[@key].to_s
-            event[new_key] = entry[@value]
-          else
-            new_key = @prefix.to_s + entry.instance_variable_get("@" + @key)
-            event[new_key] = entry.instance_variable_get("@" + @value)
-          end # if is hash
+          @logger.warn(entry.to_a.to_s)
+          if @access_by_name
+
+            if entry.is_a?(::Hash) # see spec file: test case 1 
+              if !entry[@key].nil?
+                new_key = @prefix.to_s + entry[@key].to_s
+                value = entry[@value]
+                @logger.info("Adding new field " + new_key + " with value " + value)
+              else # might be a symbol then and we need to convert our keys to :keys
+                new_key = @prefix.to_s + entry[@key.to_sym].to_s
+                value = entry[@value.to_sym]
+                @logger.info("Adding new field " + new_key + ", value " + value)
+              end
+              event.set(new_key, value)
+            else # it's an object of some unknown class.
+              @logger.warn("Data structure not supported. " + entry.inspect.to_s) 
+            end # if is hash
+
+          else # access by position, no key / value names provided
+            
+            if entry.is_a?(::Hash)  # see spec file: test case 2
+              new_key = @prefix.to_s + entry.keys[0].to_s
+              event.set(new_key, entry.values[0])
+
+            else # it's an object of some unknown class. 
+              @logger.warn("Data structure not supported. " + entry.inspect.to_s)
+            end # if is hash
+
+          end # acess type
         rescue 
           @logger.debug("Could not find key " + @key + " in incoming data, please check your config. ")
         end
-      end # do     
+      end # do
+      if @remove_source
+        event.remove(@source)
+      end
+    
     end # if input.nil? 
   end # def filter  
 end # class LogStash::Filters::List2fields

data/logstash-filter-list2fields.gemspec

@@ -1,13 +1,12 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-list2fields'
-  s.version         = '0.1.0'
+  s.version         = '0.1.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "$summary"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
-  s.authors         = ["Elastic"]
-  s.email           = 'info@elastic.co'
-  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.authors         = ["Inga Feick"]
+  s.email           = 'inga.feick@trivago.com'
   s.require_paths = ["lib"]
 
   # Files
@@ -20,7 +19,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/list2filters_spec.rb

@@ -4,64 +4,132 @@ require "logstash/filters/list2fields"
 
 describe LogStash::Filters::List2fields do
 
-  let(:plugin) { LogStash::Filters::List2fields.new("source" => "message") }
-  
-  before do
-    plugin.register
-  end
-  
-  context "when a prefix is set" do
-
-    let(:plugin_prefix) { LogStash::Filters::List2fields.new("source" => "message", "prefix" => "l2f_") } # TODO there's got to be a more beautiful way to do this 
-    let(:event) { LogStash::Event.new("cheese" => "chili", "message" => [{"key"=>"foo","value"=>"bar"},{"key"=>"cheese","value"=>"bacon"}]) } 
+  # Test cases: 1     [ { "key" => "animal", "value" => "horse"} , {"key" => "food", "value" => "bacon"} ]          list of hashes with splitted fields for key and value
+  # Test cases: 2     [ { "animal" => "horse"} , {"food" => "bacon"} ]                                              list of hashes with k => v structure
+
+  context "without prefix" do
+
+    let(:plugin) { LogStash::Filters::List2fields.new("source" => "message") }
+    
     before do
-      plugin_prefix.filter(event)
+      plugin.register
     end
+    
+    context "when the source field is empty" do
+
+      let(:event) { LogStash::Event.new() }
+      it "should not throw an exception" do
+        expect {
+          plugin.filter(event)
+        }.not_to raise_error
+      end # it
+    end # context
+
+    context "when the source field is not iterable" do
+
+      let(:event) { LogStash::Event.new("message" => "i_am_not_iterable" ) }
+      it "should not throw an exception" do
+        expect {
+          plugin.filter(event)
+        }.not_to raise_error
+      end # it
+    end # context
+
+    context "when remove_source is set to true" do
+
+      let(:event) { LogStash::Event.new("message" => [{"key"=>"foo","value"=>"bar"}]) } 
+      before do
+        plugin.filter(event)
+      end
 
-    it "should have new fields with the prefix in the key" do
-      expect(event["l2f_foo"]).to eq("bar")
-      expect(event["l2f_cheese"]).to eq("bacon")
-    end # it
+      it "should remove the input field" do       
+        expect(event.get("message")).to be_nil
+      end # it
+    end # context
 
-    it "should not overwrite existing fields" do
-      expect(event["cheese"]).to eq("chili")
-    end # it
-  end # context
+    context "when remove_source is set to false" do
+
+      let(:event) { LogStash::Event.new("message" => [{"key"=>"foo","value"=>"bar"}]) } 
+      let(:plugin) { LogStash::Filters::List2fields.new("source" => "message", "remove_source" => false) }
+    
+      before do
+        plugin.register
+        plugin.filter(event)
+      end
 
+      it "should not remove the input field" do       
+        expect(event.get("message")).not_to be_empty
+      end # it
+    end # context
 
-  context "when the source field is empty" do
 
-    let(:event) { LogStash::Event.new() }
-    it "should not throw an exception" do
-      expect {
+    context "operates on a list of hashes with splitted key and value entries (using names) (testcase 1)" do
+      let(:event) { LogStash::Event.new("cheese" => "chili", "message" => [{"key"=>"foo","value"=>"bar"},{"key"=>"cheese","value"=>"gorgonzola"}]) } 
+      let(:plugin) { LogStash::Filters::List2fields.new("source" => "message", "key" => "key", "value" => "value") }
+      before do
+        plugin.register
         plugin.filter(event)
-      }.not_to raise_error
-    end # it
-  end # context
+      end
 
-  context "when the source field is not iterable" do
+      it "should have new fields" do
+        expect(event.get("foo")).to eq("bar")
+        expect(event.get("cheese")).to eq("gorgonzola")
+      end # it
+    end # context
 
-    let(:event) { LogStash::Event.new("message" => "i_am_not_iterable" ) }
-    it "should not throw an exception" do
-      expect {
+
+    context "operates on a list of hashes with key and value in one tuple (testcase 2)." do
+      let(:event) { LogStash::Event.new("cheese" => "chili", "message" => [{"foo"=>"bar"},{"cheese"=>"gorgonzola"}]) } 
+      before do
+        plugin.register
         plugin.filter(event)
-      }.not_to raise_error
-    end # it
-  end # context
+      end
 
+      it "should have new fields" do
+        expect(event.get("foo")).to eq("bar")
+        expect(event.get("cheese")).to eq("gorgonzola")
+      end # it
+    end # context
 
-  context "when there is a nested list" do
+  end # context no prefix set
 
-    let(:event) { LogStash::Event.new("message" => [{"key"=>"foo","value"=>"bar"},{"key"=>"cheese","value"=>"bacon"}]) } 
-    before do
-      plugin.filter(event)
-    end
+  context "with prefix " do
+
+    context "operates on a list of hashes with splitted key and value entries (using names) (testcase 1)" do
+      let(:plugin) { LogStash::Filters::List2fields.new("source" => "message", "prefix" => "l2f_", "key" => "key", "value" => "value") } 
+      let(:event) { LogStash::Event.new("cheese" => "chili", "message" => [{"key"=>"foo","value"=>"bar"},{"key"=>"cheese","value"=>"gorgonzola"}]) } 
+      before do
+        plugin.register
+        plugin.filter(event)
+      end
+
+      it "should have new fields with the prefix in the key" do
+        expect(event.get("l2f_foo")).to eq("bar")
+        expect(event.get("l2f_cheese")).to eq("gorgonzola")
+      end # it
+
+      it "should not overwrite existing fields" do
+        expect(event.get("cheese")).to eq("chili")
+      end # it
+    end # context
+
+
+
+    context "operates on a list of hashes with key and value in one tuple (testcase 2)." do
+      let(:plugin) { LogStash::Filters::List2fields.new("source" => "message", "prefix" => "l2f_") } 
+      let(:event) { LogStash::Event.new("cheese" => "chili", "message" => [{"foo"=>"bar"},{"cheese"=>"gorgonzola"}]) } 
+      before do
+        plugin.register
+        plugin.filter(event)
+      end
+
+      it "should have new fields with the prefix in the key" do
+        expect(event.get("l2f_foo")).to eq("bar")
+        expect(event.get("l2f_cheese")).to eq("gorgonzola")
+      end # it
+    end # context
 
-    it "should have new fields" do       
-      expect(event["foo"]).to eq("bar")
-      expect(event["cheese"]).to eq("bacon")
-    end # it
+  end # context prefix set
 
-  end # context  
 end # describe
 

metadata

@@ -1,51 +1,51 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-list2fields
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.3
 platform: ruby
 authors:
-- Elastic
+- Inga Feick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-25 00:00:00.000000000 Z
+date: 2016-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+        version: '1.60'
+    - - <=
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.60'
+    - - <=
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
-email: info@elastic.co
+email: inga.feick@trivago.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -61,7 +61,7 @@ files:
 - spec/filters/list2filters_spec.rb
 - vendor/GeoIPASNum-2014-02-12.dat
 - vendor/GeoLiteCity-2013-01-18.dat
-homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+homepage:
 licenses:
 - Apache License (2.0)
 metadata: