fluent-plugin-flatten-hash 0.0.3 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f772de761e1745118e528735b2eef31caec0f1d8
+  data.tar.gz: e40c3922daafb54256c362264813ad8f648311e2
+SHA512:
+  metadata.gz: ed1d12c01bdfbc22def38e47151899baf359fb82edb0269d99b2c7c55dbaee805f3b23512fb152c4d69aeb6f75804c0f99effc89de94088d4049edc9bbd833b3
+  data.tar.gz: 223aebc69f80a14abe9ac4cd5e1e05f2f0e06e3f2de3726cded8c987bda8f0e76f18f68cb2b828576b40fa1344ca99e1afbb5db4cc446c702ec0ed64ae52c51f

data/README.md

@@ -1,4 +1,4 @@
-# fluent-plugin-flatten-hash
+# fluent-plugin-flatten-hash, a plugin for [Fluentd](http://fluentd.org)
 
 A fluentd plugin to flatten nested hash structure as a flat record with unique keys generated by its path for each values.
 
@@ -18,6 +18,10 @@ Or install it yourself as:
 
 ## Configuration
 
+fluent-plugin-flatten-hash supports both Output and Filter plugin.
+
+### Output
+
 You can set a configuration like below:
 
 ```
@@ -41,7 +45,7 @@ In this configuration, if you get a following nested/complex message:
         }
       }
     }
-  }
+  },
   "days":[
     "2013/08/24",
     "2013/08/25"
@@ -54,12 +58,27 @@ The message is flattened like below:
 ```js
 {
   "message_today":"good day",
-  "message_tommorow_is_a_bad":"day"
-  "days_0" => "2013/08/24",
-  "days_1" => "2013/08/25"
+  "message_tommorow_is_a_bad":"day",
+  "days_0":"2013/08/24",
+  "days_1":"2013/08/25"
 }
 ```
 
+### Filter
+
+You can set a configuration like below:
+
+```
+<filter message>
+  type flatten_hash
+  separator _
+</filter>
+
+<match message>
+  type stdout
+</match>
+```
+
 ## Contributing
 
 1. Fork it
@@ -84,4 +103,4 @@ The message is flattened like below:
   <tr>
     <td>License</td><td>MIT License</td>
   </tr>
-</table>
+</table>

data/fluent-plugin-flatten-hash.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-flatten-hash"
-  spec.version       = "0.0.3"
+  spec.version       = "0.1.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.description   = %q{A fluentd plugin to flatten nested hash structure as a flat record}
@@ -17,6 +17,6 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "fluentd", "~> 0.10.0"
+  spec.add_dependency "fluentd", ["> 0.10.0", "< 0.14.0"]
   spec.add_development_dependency "rake"
 end

data/lib/fluent/plugin/filter_flatten_hash.rb

@@ -0,0 +1,18 @@
+module Fluent
+  class FlattenHashFilter < Filter
+    Plugin.register_filter('flatten_hash', self)
+
+    require_relative 'util'
+    include FlattenHashUtil
+
+    config_param :separator, :string, :default => '.'
+
+    def configure(conf)
+      super
+    end
+
+    def filter(tag, time, record)
+      flatten_record(record, [])
+    end
+  end if defined?(Filter)
+end

data/lib/fluent/plugin/out_flatten_hash.rb

@@ -3,6 +3,9 @@ module Fluent
     include Fluent::HandleTagNameMixin
     Fluent::Plugin.register_output('flatten_hash', self)
 
+    require_relative 'util'
+    include FlattenHashUtil
+
     config_param :tag, :string, :default => nil
     config_param :separator, :string, :default => '.'
 
@@ -31,22 +34,5 @@ module Fluent
       end
       chain.next
     end
-
-    private
-    def flatten_record(record, prefix)
-      ret = {}
-      if record.is_a? Hash
-        record.each { |key, value|
-          ret.merge! flatten_record(value, prefix + [key.to_s])
-        }
-      elsif record.is_a? Array
-        record.each_with_index { |elem, index|
-          ret.merge! flatten_record(elem, prefix + [index.to_s])
-        }
-      else
-        return {prefix.join(@separator) => record}
-      end
-      ret
-    end
   end
 end

data/lib/fluent/plugin/util.rb

@@ -0,0 +1,19 @@
+module Fluent
+  module FlattenHashUtil
+    def flatten_record(record, prefix)
+      ret = {}
+      if record.is_a? Hash
+        record.each { |key, value|
+          ret.merge! flatten_record(value, prefix + [key.to_s])
+        }
+      elsif record.is_a? Array
+        record.each_with_index { |elem, index|
+          ret.merge! flatten_record(elem, prefix + [index.to_s])
+        }
+      else
+        return {prefix.join(@separator) => record}
+      end
+      ret
+    end
+  end
+end

data/test/helper.rb

@@ -12,17 +12,3 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
-
-require 'fluent/plugin/out_flatten_hash'
-
-class Test::Unit::TestCase
-end

data/test/plugin/test_filter_flatten_hash.rb

@@ -0,0 +1,57 @@
+require 'helper'
+require 'fluent/plugin/filter_flatten_hash'
+
+class FlattenHashFilterTest < Test::Unit::TestCase
+  include Fluent
+
+  BASE_CONFIG = %[
+    type flatten_hash
+  ]
+  CONFIG = BASE_CONFIG + %[
+    add_tag_prefix flattened
+  ]
+
+  def setup
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  def create_driver(conf = '')
+    Test::FilterTestDriver.new(FlattenHashFilter).configure(conf, true)
+  end
+
+  def test_flatten_record
+    d = create_driver
+    es = Fluent::MultiEventStream.new
+    now = Fluent::Engine.now
+
+    d.run do
+      d.emit({'message' => {'foo' => 'bar'}})
+      d.emit({"message" => {'foo' => 'bar', 'hoge' => 'fuga'}})
+      d.emit({"message" => {'nest' => {'foo' => 'bar'}}})
+      d.emit({"message" => {'nest' => {'nest' => {'foo' => 'bar'}}}})
+      d.emit({"message" => {'array' => ['foo', 'bar']}})
+      d.emit({"message" => {'array' => [{'foo' => 'bar'}, {'hoge' => 'fuga'}]}})
+    end
+
+    assert_equal [
+      {"message.foo" => "bar"},
+      {"message.foo" => "bar", "message.hoge" => "fuga"},
+      {"message.nest.foo" => "bar"},
+      {"message.nest.nest.foo" => "bar"},
+      {"message.array.0" => "foo", "message.array.1" => "bar"},
+      {"message.array.0.foo" => "bar", "message.array.1.hoge" => "fuga"},
+    ], d.filtered_as_array.map{|x| x[2]}
+  end
+
+  def test_separator
+    d = create_driver CONFIG + %[separator /]
+
+    d.run do
+      d.emit({"message" => {'nest' => {'foo' => 'bar'}}})
+    end
+    assert_equal [
+      {"message/nest/foo" => "bar"},
+    ], d.filtered_as_array.map{|x| x[2]}
+  end
+end

data/test/plugin/test_out_flatten_hash.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/plugin/out_flatten_hash'
 
 class FlattenHashOutputTest < Test::Unit::TestCase
   def setup

metadata

@@ -1,46 +1,47 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flatten-hash
 version: !ruby/object:Gem::Version
-  version: 0.0.3
-  prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-14 00:00:00.000000000 Z
+date: 2014-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">"
       - !ruby/object:Gem::Version
         version: 0.10.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">"
       - !ruby/object:Gem::Version
         version: 0.10.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: A fluentd plugin to flatten nested hash structure as a flat record
@@ -50,40 +51,43 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - fluent-plugin-flatten-hash.gemspec
+- lib/fluent/plugin/filter_flatten_hash.rb
 - lib/fluent/plugin/out_flatten_hash.rb
+- lib/fluent/plugin/util.rb
 - test/helper.rb
+- test/plugin/test_filter_flatten_hash.rb
 - test/plugin/test_out_flatten_hash.rb
 homepage: https://github.com/sabottenda/fluent-plugin-flatten-hash
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A fluentd plugin to flatten nested hash structure as a flat record
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_flatten_hash.rb
 - test/plugin/test_out_flatten_hash.rb