fluent-plugin-flatten 0.0.2 → 0.0.3

data/README.md

@@ -4,17 +4,20 @@
 
 ### FlattenOutput
 
-Fluentd output plugin to flatten JSON-formatted string values in records to top level key/value-s.
+Fluentd plugin to extract values for nested key paths and re-emit them as flat tag/record pairs.
 
 ## Synopsis
 
-When you have a config as below:
+Imagin you have a config as below:
 
 ```
 <match test.**>
   type flatten
+
   key  foo
-  add_tag_prefix flattened.
+  add_tag_prefix    flattened.
+  remove_tag_prefix test.
+  inner_key         value_for_flat_key
 </match>
 ```
 
@@ -27,20 +30,20 @@ And you feed such a value into fluentd:
 }
 ```
 
-Then you'll get:
+Then you'll get re-emmited tag/record-s below:
 
 ```
-"flattened.test" => {
-  "foo"  => '{"bar" : {"qux" : "quux", "hoe" : "poe" }, "baz" : "bazz" }',
-  "hoge" => "fuga",
-
-  "foo.bar.qux" => "quux",
-  "foo.bar.hoe" => "poe",
-  "foo.baz"     => "bazz"
-}
+"flattened.foo.bar.qux" => { "value_for_flat_key" => "quux" }
+"flattened.foo.bar.hoe" => { "value_for_flat_key" => "poe"  }
+"flattened.foo.baz"     => { "value_for_flat_key" => "bazz" }
 ```
 
-That is, JSON-formatted string in the value of the key `foo` is flattened and now put into the top level of the hash.
+That is to say:
+
+  1. The JSON-formatted string in the value related to the key `foo` is inflated to a `Hash`.
+  2. The values are extracted as to be related to the nested key paths (`foo.bar.baz`).
+  3. This plugin re-emits them as new tag/record pairs.
+  4. Key/value pairs whose keys don't match `foo` are ignored (`"hoge" => "fuga"`).
 
 ## Configuration
 
@@ -55,6 +58,10 @@ These params are included from `Fluent::HandleTagNameMixin`. See that code for d
 
 You must add at least one of these params.
 
+### inner_key
+
+This plugin sets `value` for this option as a default if it's not set.
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/fluent-plugin-flatten.gemspec

@@ -1,11 +1,11 @@
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-flatten'
-  gem.version       = '0.0.2'
+  gem.version       = '0.0.3'
   gem.authors       = ['Kentaro Kuribayashi']
   gem.email         = ['kentarok@gmail.com']
   gem.homepage      = 'http://github.com/kentaro/fluent-plugin-flatten'
-  gem.description   = %q{Fluentd plugin to flatten JSON-formatted string values to top level key/value-s.}
-  gem.summary       = %q{Fluentd plugin to flatten JSON-formatted string values to top level key/value-s.}
+  gem.description   = %q{Fluentd plugin to extract values for nested key paths and re-emit them as flat tag/record pairs.}
+  gem.summary       = %q{Fluentd plugin to extract values for nested key paths and re-emit them as flat tag/record pairs.}
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/lib/fluent/plugin/out_flatten.rb

@@ -4,27 +4,34 @@ module Fluent
   class FlattenOutput < Output
     include Fluent::HandleTagNameMixin
     class Error < StandardError; end
+
     Fluent::Plugin.register_output('flatten', self)
 
-    config_param :key, :string
+    config_param :key,       :string
+    config_param :inner_key, :string, :default => 'value'
 
     def configure(conf)
       super
 
-      if !self.remove_tag_prefix && !self.remove_tag_suffix && !self.add_tag_prefix && !self.add_tag_suffix
-        raise ConfigError, "out_flatten: Set remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
+      if (
+          !remove_tag_prefix &&
+          !remove_tag_suffix &&
+          !add_tag_prefix    &&
+          !add_tag_suffix
+      )
+        raise ConfigError, "out_flatten: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set"
       end
     end
 
     def emit(tag, es, chain)
       es.each do |time, record|
-        _tag      = tag.clone
         flattened = flatten(record)
-        filter_record(_tag, time, flattened)
-        if tag != _tag
-          Engine.emit(_tag, time, flattened)
-        else
-          $log.warn "Drop record #{record} tag '#{tag}' was not replaced. Can't emit record, cause infinity looping. Set remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix correctly."
+
+        flattened.each do |keypath, value|
+          tag_with_keypath = [tag.clone, keypath].join('.')
+          filter_record(tag_with_keypath, time, value)
+
+          Engine.emit(tag_with_keypath, time, value)
         end
       end
 
@@ -32,31 +39,34 @@ module Fluent
     end
 
     def flatten(record)
-      if record.has_key?(@key)
-        hash   = JSON.parse(record[@key])
-        record = record.merge(_flatten(@key, hash))
-      end
+      flattend = {}
 
-      record
-    end
+      if record.has_key?(key)
+        hash      = JSON.parse(record[key])
+        processor = lambda do |root, hash|
+          unless hash.is_a?(Hash)
+            raise Error.new('The value to be flattened must be a Hash: #{hash}')
+          end
 
-    def _flatten(root, hash)
-      unless hash.is_a?(Hash)
-        raise Error.new('The value to be flattened must be a Hash: #{hash}')
-      end
+          flattened = {}
 
-      flattened = {}
-      hash.each do |path, value|
-        key = [root, path].join('.')
+          hash.each do |path, value|
+            keypath = [root, path].join('.')
 
-        if value.is_a?(String)
-          flattened[key] = value
-        else
-          flattened = flattened.merge(_flatten(key, value))
+            if value.is_a?(Hash)
+              flattened = flattened.merge(processor.call(keypath, value))
+            else
+              flattened[keypath] = { inner_key => value }
+            end
+          end
+
+          flattened
         end
+
+        flattend  = processor.call(key, hash)
       end
 
-      flattened
+      flattend
     end
   end
 end

data/test/plugin/test_out_flatten.rb

@@ -5,19 +5,57 @@ class FlattenOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  CONFIG = %[
-    key foo
-    add_tag_prefix flattened.
+  DEFAULT_CONFIG = %[
+    key               foo
+    add_tag_prefix    flattened.
+    remove_tag_prefix test.
   ]
 
-  def create_driver(conf = CONFIG, tag = 'test')
+  def create_driver(conf = DEFAULT_CONFIG, tag = 'test')
     Fluent::Test::OutputTestDriver.new(Fluent::FlattenOutput, tag).configure(conf)
   end
 
   def test_configure
+    # when `inner_key` option is not set
+    d1 = create_driver
+
+    assert_equal 'foo',                d1.instance.key
+    assert_equal 'flattened.',         d1.instance.add_tag_prefix
+    assert_equal /^test\./,            d1.instance.remove_tag_prefix
+    assert_equal 'value',              d1.instance.inner_key          # default value
+
+    # when `inner_key` is set
+    d2 = create_driver(%[
+      key               foo
+      add_tag_prefix    flattened.
+      remove_tag_prefix test.
+      inner_key         value_for_flat_key
+    ])
+
+    assert_equal 'foo',                d2.instance.key
+    assert_equal 'flattened.',         d2.instance.add_tag_prefix
+    assert_equal /^test\./,            d2.instance.remove_tag_prefix
+    assert_equal 'value_for_flat_key', d2.instance.inner_key
+
+    # when mandatory keys not set
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        key        foo
+        inner_key  value_for_keypath
+      ])
+    end
+  end
+
+  def test_flatten
     d = create_driver
-    assert_equal 'foo', d.instance.key
-    assert_equal 'flattened.', d.instance.add_tag_prefix
+
+    flattened = d.instance.flatten({ 'foo' => '{"bar" : "baz"}', 'hoge' => 'fuga' })
+    assert_equal({ 'foo.bar' => { 'value' => 'baz' } }, flattened)
+
+    # when not hash value is passed
+    assert_raise(Fluent::FlattenOutput::Error) do
+      d.instance.flatten({ 'foo' => '["bar", "baz"]' })
+    end
   end
 
   def test_emit
@@ -29,15 +67,22 @@ class FlattenOutputTest < Test::Unit::TestCase
     end
     emits = d.emits
 
-    assert_equal      3, emits[0][2].count
-    assert_equal  'baz', emits[0][2]['foo.bar']
+    assert_equal 4, emits.count
+
+    # ["flattened.foo.bar", 1354689632, {"value"=>"baz"}]
+    assert_equal     'flattened.foo.bar', emits[0][0]
+    assert_equal                   'baz', emits[0][2]['value']
+
+    # ["flattened.foo.bar.qux", 1354689632, {"value"=>"quux"}]
+    assert_equal 'flattened.foo.bar.qux', emits[1][0]
+    assert_equal                  'quux', emits[1][2]['value']
 
-    assert_equal      5, emits[1][2].count
-    assert_equal 'quux', emits[1][2]['foo.bar.qux']
-    assert_equal  'poe', emits[1][2]['foo.bar.hoe']
-    assert_equal 'bazz', emits[1][2]['foo.baz']
+    # ["flattened.foo.bar.hoe", 1354689632, {"value"=>"poe"}]
+    assert_equal 'flattened.foo.bar.hoe', emits[2][0]
+    assert_equal                   'poe', emits[2][2]['value']
 
-    assert_equal 'flattened.test', emits[0][0]
-    assert_equal 'flattened.test', emits[1][0]
+    # ["flattened.foo.bar.baz", 1354689632, {"value"=>"bazz"}]
+    assert_equal     'flattened.foo.baz', emits[3][0]
+    assert_equal                  'bazz', emits[3][2]['value']
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flatten
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-29 00:00:00.000000000 Z
+date: 2012-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -59,7 +59,8 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Fluentd plugin to flatten JSON-formatted string values to top level key/value-s.
+description: Fluentd plugin to extract values for nested key paths and re-emit them
+  as flat tag/record pairs.
 email:
 - kentarok@gmail.com
 executables: []
@@ -98,7 +99,8 @@ rubyforge_project:
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Fluentd plugin to flatten JSON-formatted string values to top level key/value-s.
+summary: Fluentd plugin to extract values for nested key paths and re-emit them as
+  flat tag/record pairs.
 test_files:
 - test/plugin/test_out_flatten.rb
 - test/test_helper.rb