fluent-plugin-flatten-hash 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d6ccf5a400f383fccdd8e01058f5d9c1a3d215b0
-  data.tar.gz: 792eb0e870326f0952c971aabd71c99f70cb0d22
+  metadata.gz: bd341b3cebe6a2b4dc68dbdc5b0e748acc25de57
+  data.tar.gz: 0c8ef3cc6db0d4324cef5a85a07340db96272a46
 SHA512:
-  metadata.gz: fcf04cc0c46dfb68cd1fa358ae1e2239ed09bd43b6776b540486a33688011d42aabd02958c231e23c755d42927723951827d8b563aa96dac23bb6175e3712bbf
-  data.tar.gz: 06df261413e860fa7f0827fdbc0222057c6ec784a5caf631fec771b56a80ced3781c22ff46b639b8fcad898830f8edd3b7e121d7a95080fae847078428f40268
+  metadata.gz: c8a52a3dc1e4f390bbc7a8b1d1edbe19b78379b683d4682c5f2c340c8719d51cdc468fbf6078dcd9e85311d27a1cbb68f2e1340a2c40ea2270af4d36a80e36f2
+  data.tar.gz: 0b64864c3082f235285e90baf3d81a1bc7a2ab451d4673b04a350dd07a44325d9bf3e8e4765eb477a965961a8fc3102929b24aabaf58151c1c4128735a3570ac

data/README.md

@@ -66,6 +66,28 @@ The message is flattened like below:
 }
 ```
 
+In order to prevent arrays from being indexed, you can use a configuration like below:
+
+```
+<match message>
+  type flatten_hash
+  add_tag_prefix flattened.
+  separator _
+  flatten_array false
+</match>
+```
+
+Using the same input, you'll instead end up with a message flattened like below:
+
+```js
+{
+  "message_today":"good day",
+  "message_tommorow_is_a_bad":"day",
+  "days":["2013/08/24","2013/08/25"]
+}
+```
+
+
 ### Filter
 
 You can set a configuration like below:

data/fluent-plugin-flatten-hash.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-flatten-hash"
-  spec.version       = "0.3.0"
+  spec.version       = "0.4.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.description   = %q{A fluentd plugin to flatten nested hash structure as a flat record}

data/lib/fluent/plugin/filter_flatten_hash.rb

@@ -6,7 +6,8 @@ module Fluent
     include FlattenHashUtil
 
     config_param :separator, :string, :default => '.'
-
+    config_param :flatten_array, :bool, :default => true
+    
     def configure(conf)
       super
     end

data/lib/fluent/plugin/flatten_hash_util.rb

@@ -7,9 +7,13 @@ module Fluent
           ret.merge! flatten_record(value, prefix + [key.to_s])
         }
       elsif record.is_a? Array
-        record.each_with_index { |elem, index|
-          ret.merge! flatten_record(elem, prefix + [index.to_s])
-        }
+        if @flatten_array
+          record.each_with_index { |elem, index|
+            ret.merge! flatten_record(elem, prefix + [index.to_s])
+          }
+        else
+          return {prefix.join(@separator) => record}
+        end
       else
         return {prefix.join(@separator) => record}
       end

data/lib/fluent/plugin/out_flatten_hash.rb

@@ -8,6 +8,7 @@ module Fluent
 
     config_param :tag, :string, :default => nil
     config_param :separator, :string, :default => '.'
+    config_param :flatten_array, :bool, :default => true
 
     def initialize
       super

data/test/plugin/test_filter_flatten_array.rb

@@ -0,0 +1,47 @@
+require 'helper'
+require 'fluent/plugin/filter_flatten_hash'
+
+class FlattenHashFlattenArrayFilterTest < Test::Unit::TestCase
+  include Fluent
+
+  BASE_CONFIG = %[
+    type flatten_hash
+  ]
+  CONFIG = BASE_CONFIG + %[
+    add_tag_prefix flattened
+    flatten_array false
+  ]
+
+  def setup
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  def create_driver(conf = '')
+    Test::FilterTestDriver.new(FlattenHashFilter).configure(conf, true)
+  end
+
+  def test_flatten_record_flatten_array_false
+    d = create_driver CONFIG
+    es = Fluent::MultiEventStream.new
+    now = Fluent::Engine.now
+
+    d.run do
+      d.emit({'message' => {'foo' => 'bar'}})
+      d.emit({"message" => {'foo' => 'bar', 'hoge' => 'fuga'}})
+      d.emit({"message" => {'nest' => {'foo' => 'bar'}}})
+      d.emit({"message" => {'nest' => {'nest' => {'foo' => 'bar'}}}})
+      d.emit({"message" => {'array' => ['foo', 'bar']}})
+      d.emit({"message" => {'array' => [{'foo' => 'bar'}, {'hoge' => 'fuga'}]}})
+    end
+
+    assert_equal [
+      {"message.foo" => "bar"},
+      {"message.foo" => "bar", "message.hoge" => "fuga"},
+      {"message.nest.foo" => "bar"},
+      {"message.nest.nest.foo" => "bar"},
+      {"message.array"=>["foo", "bar"]},
+      {"message.array"=>[{"foo"=>"bar"}, {"hoge"=>"fuga"}]},
+    ], d.filtered_as_array.map{|x| x[2]}
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flatten-hash
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-04 00:00:00.000000000 Z
+date: 2016-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -70,6 +70,7 @@ files:
 - lib/fluent/plugin/flatten_hash_util.rb
 - lib/fluent/plugin/out_flatten_hash.rb
 - test/helper.rb
+- test/plugin/test_filter_flatten_array.rb
 - test/plugin/test_filter_flatten_hash.rb
 - test/plugin/test_out_flatten_hash.rb
 homepage: https://github.com/kazegusuri/fluent-plugin-flatten-hash
@@ -98,5 +99,6 @@ specification_version: 4
 summary: A fluentd plugin to flatten nested hash structure as a flat record
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_flatten_array.rb
 - test/plugin/test_filter_flatten_hash.rb
 - test/plugin/test_out_flatten_hash.rb