logstash-filter-denormalize 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ac20ed323d0c620f7b31525d2cb1e77461415977
+  data.tar.gz: 4bbe769bd1d7a503e793ee75bce0944cc307a1d8
+SHA512:
+  metadata.gz: 0b4c540717223854b8a7542f8d80b26195c6bce774a0ed1646bd2a1e412f16ee9e751b4eab50e199926b18dae8cc1d515bb3925d435bb0a8d5012422ce7ae811
+  data.tar.gz: 22c43a69c9f0ec4307fec2942933cdfb364c50382d4fc6de1b51fff45f0250135681a3ea4187fe7b5b3bda41124832a3de8bbb4fe12266186efe9a1809e6ad7d

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Inga Feick (ingafeick)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 trivago <http://www.trivago.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+trivago GmbH
+Copyright 2012-2015 trivago 
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,88 @@
+# Logstash denormalize Filter Plugin
+
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Installation
+
+You can download the plugin from [rubygems](https://rubygems.org/gems/logstash-filter-denormalize) and install it from your logstash home directory like so:
+
+	bin/plugin install logstash-filter-denormalize-$VERSION.gem
+
+## Versions and compatibility
+
+Versions below 0.1.2 are compatible with logstash 2.x. Version 0.1.2 and later are compatible with logstash 5.
+
+## Purpose
+
+This filter will denormalize an event with a field of n values in an array into n differents events, which are the same except for the values in the array field. Each event will contain one of the array values in that field.  
+Basically, it behaves like the [split filter](https://www.elastic.co/guide/en/logstash/current/plugins-filters-split.html) but it splits on objects, not string sections.
+
+### Examples
+  
+	Input event: 
+	{
+		"host" => "machine4711"
+		"services" => ["elasticsearch","logstash","collectd"]
+		"ip" => "10.1.2.3"
+	}
+
+Output events:
+
+	{
+		"host" => "machine4711"
+		"services" => "elasticsearch"
+		"ip" => "10.1.2.3"
+	}
+	{
+		"host" => "machine4711"
+		"services" => "logstash"
+		"ip" => "10.1.2.3"
+	}
+	{
+		"host" => "machine4711"
+		"services" => "collectd"
+		"ip" => "10.1.2.3"
+	}
+
+If the field, upon which the event is to be splitted, contains an array of key=>value nature, then those keys will be used in the new events:
+
+Input event: 
+	{
+		"host" => "machine4711"
+		"attributes" => {
+						"ram" => "32g",
+						"cores" => 8
+						}
+		"ip" => "10.1.2.3"
+	}
+
+Output events:
+	{
+		"host" => "machine4711"
+		"ram" => "32g"
+		"ip" => "10.1.2.3"
+	}
+	{
+		"host" => "machine4711"
+		"cores" => 8
+		"ip" => "10.1.2.3"
+	}
+
+## Configuration
+
+* source 	: name of the field by which to denormalize. For each entry in this field (which should have an array value) a new record will be created, containing all the other fields. 
+* target	: new key for the field by which you splitted the event. This is only effective if the splitted field didn't have keys on its own.
+
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/denormalize.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# TODO docu
+class LogStash::Filters::Denormalize < LogStash::Filters::Base
+  
+  config_name "denormalize"
+
+  # The name of the field which contains the array or hash value(s)
+  config :source, :validate => :string
+
+  # New name for the splitted field in the new events, if it is not a hash
+  config :target, :validate => :string, :default => ""
+
+  # Delete the original event after it has been splitted
+  config :delete_original, :validate => :boolean, :default => true
+
+  public
+  def register
+    @list_target = (@target.nil? || @target.empty?) ? @source : @target # if no target name is provided: keep original name.
+
+  end # def register
+
+  public
+  def filter(event)
+    input = event.get(@source)
+    if !input.nil?  
+      if input.is_a?(::Hash) # if it's a hash then let's take the keys from the original data
+        input.each do |key, value|
+          target = (!@target.nil? && !@target.empty?) ? @target : key
+          event_split = event.clone
+          event_split.set(target, value)
+          filter_matched(event_split) 
+          yield event_split
+          if @delete_original
+            event.cancel # TODO why is this here and not outside the loop?
+          end
+        end # do
+      elsif (input.is_a? Enumerable)
+        input.each do |value|
+          # magic(event, @target, value)
+          event_split = event.clone
+          event_split.set(@list_target, value)
+          filter_matched(event_split)
+          yield event_split
+          if @delete_original
+            event.cancel # TODO why is this here and not outside the loop?
+          end
+        end # do    
+      else
+        @logger.debug("Not iterable: field " + @source + " with value " + input.to_s)
+      end
+    else
+       @logger.debug("Nil: field " + @source)
+    end # if input.nil? 
+  end # def filter  
+
+
+end # class LogStash::Filters::List2fields
+
+

data/logstash-filter-denormalize.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-filter-denormalize'
+  s.version         = '0.1.2'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "$summary"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Inga Feick"]
+  s.email           = 'inga.feick@trivago.com'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/filters/denormalize_spec.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/denormalize"
+require "logstash/event"
+
+describe LogStash::Filters::Denormalize do
+
+  context "when field is a list" do
+    it "should not raise exception" do
+      filter = LogStash::Filters::Denormalize.new({"source" => "my_array"})
+      event = LogStash::Event.new("my_array" => ["cat","dog"])
+      
+      expect {filter.filter(event).length}.to eq(2)
+    end
+  end
+
+
+=begin
+
+  context "when field is nil" do
+    it "should not raise exception" do
+      filter = LogStash::Filters::Denormalize.new({"source" => "my_array"})
+      event = LogStash::Event.new("my_array" => nil)
+      expect {filter.filter(event)}.not_to raise_error
+    end
+  end
+
+ describe "array input" do
+    config <<-CONFIG
+      filter {
+        denormalize { "source" => "message"}
+      }
+    CONFIG
+
+    sample ["cheese", "bacon"] do
+      puts subject.inspect # TODO
+      insist { subject.length } == 2
+      insist { subject[0]["message"] } == "cheese"
+      insist { subject[1]["message"] } == "bacon"
+    end
+  end
+
+  context "when field is not iterable" do
+    it "should not raise exception" do
+      filter = LogStash::Filters::Denormalize.new({"source" => "my_array"})
+      event = LogStash::Event.new("my_array" => "ipsum lorem")
+      expect {filter.filter(event)}.not_to raise_error
+    end
+  end
+
+
+
+  describe "change target name" do
+    config <<-CONFIG
+      filter {
+        denormalize { 
+          "source" => "message"
+          "target" => "new_key"
+        }
+      }
+    CONFIG
+
+    sample ["cheese", "bacon"] do
+      puts subject.inspect # TODO
+      insist { subject.length } == 2
+      insist { subject[0]["new_key"] } == "cheese"
+      insist { subject[1]["new_key"] } == "bacon"
+    end
+  end
+
+
+  describe "hash input" do
+    config <<-CONFIG
+      filter {
+        denormalize { "source" => "message"}
+      }
+    CONFIG
+
+    sample {"foo" => "bar", "unicorn" => "magic"} do
+      insist { subject.length } == 2
+      insist { subject[0]["foo"] }      == "bar"
+      insist { subject[1]["unicorn"] }  == "magic"
+    end
+  end
+
+  describe "hash input with target name" do
+    config <<-CONFIG
+      filter {
+        denormalize { 
+          "source" => "message"
+          "target" => "new_key"
+        }
+      }
+    CONFIG
+
+    sample {"foo" => "bar", "unicorn" => "magic"} do
+      insist { subject.length } == 2
+      insist { subject[0]["new_key"] }      == "bar"
+      insist { subject[1]["new_key"] }  == "magic"
+    end
+  end
+=end
+
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-denormalize
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Inga Feick
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-11-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - <=
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - <=
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: inga.feick@trivago.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/logstash/filters/denormalize.rb
+- logstash-filter-denormalize.gemspec
+- spec/filters/denormalize_spec.rb
+- vendor/GeoIPASNum-2014-02-12.dat
+- vendor/GeoLiteCity-2013-01-18.dat
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: $summary
+test_files:
+- spec/filters/denormalize_spec.rb