fluent-plugin-nested-hash-filter 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 60b5bb888522f56625b754d9caeadcb87d7bc44d
-  data.tar.gz: 4f44bc0f23f2c6625397886f58b7e375b4d12e41
+  metadata.gz: 8086358c385e65ba72177e5d387af2dbc59d9327
+  data.tar.gz: 00dd2d49d61abda94163664ebcc5147dc4a8439c
 SHA512:
-  metadata.gz: b58d7cfb0da91c08bbf5a60f52bcf31de08b569b5938d19bb957282cc98b8fbf5c11e8833e6ef2386284c9478057bb0efc14477f992a198a8156ec0e2fcc4524
-  data.tar.gz: 211f1c9ed5430cc2328672ca9525479607049cfcae7c86e268378e4360301b139332fdeb0589ac4d7153e01e117e658951917836a50fb8ba98922c19ff561327
+  metadata.gz: 9bf597fb8200cb91327b6e061b9749ad88f17a8af45037ff7360eef64d80acbcf3c7fd7dd65c1f5f2f8814d269e37eeb3e83f5ef5f27e9b633b4a88bd412d676
+  data.tar.gz: 9e850518c9d05ea283c89d473269301b0923dc275dc60a74f671841aa1c859f545509bbe9a42c059a16628e8a68e991359869edadec2f5dcb81bafd055a65ff4

data/README.md

@@ -1,6 +1,7 @@
 # fluent-plugin-nested-hash-filter
 
-[![Build Status](https://travis-ci.org/sugilog/fluent-plugin-nested-hash-filter.png?branch=master)](https://travis-ci.org/sugilog/fluent-plugin-nested-hash-filter.svg)
+[![Build Status](https://travis-ci.org/sugilog/fluent-plugin-nested-hash-filter.svg?branch=master)](https://travis-ci.org/sugilog/fluent-plugin-nested-hash-filter)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-nested-hash-filter.svg)](http://badge.fury.io/rb/fluent-plugin-nested-hash-filter)
 
 
 Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fluent/plugin/filter_nested_hash`.
@@ -9,23 +10,40 @@ TODO: Delete this and the text above, and describe your gem
 
 ## Installation
 
-Add this line to your application's Gemfile:
+install to your td-agent env
 
 ```ruby
-gem 'fluent-plugin-nested-hash-filter'
+td-agent-gem install fluent-plugin-nested-hash-filter
 ```
 
-And then execute:
+## Usage
 
-    $ bundle
+Add config to your `td-agent.conf`
 
-Or install it yourself as:
+**Filter**: just only convert passed hash into flatten key-value pair.
 
-    $ gem install fluent-plugin-nested-hash-filter
+```
+<filter {MATCH_PATTERN}>
+  type nested_hash
+  connector .
+</filter>
+```
 
-## Usage
+**Output**: convert passed hash into flatten key-value pair, and rename tag.
+
+```
+<match {MATCH_PATTERN}>
+  type       nested_hash
+  tag_prefix {PREFIX}
+  connector  .
+</match>
+```
+
+`tag_prefix` is required parameter to add prefix to matched tag name.
+
+`connector` is optional parameter to connect nested-keys. (default: `.`) not for tag prefix connection.
 
-TODO: Write usage instructions here
+- ex: matched tag is `access.log` and `tag_prefix` is `converted.`, then log will be passed with tag name `converted.access.log`.
 
 ## Development
 

data/fluent-plugin-nested-hash-filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-nested-hash-filter"
-  spec.version       = "0.0.3"
+  spec.version       = "0.1.0"
   spec.authors       = ["sugilog"]
   spec.email         = ["sugilog@gmail.com"]
 

data/lib/fluent/plugin/filter_nested_hash.rb

@@ -4,6 +4,8 @@ module Fluent
   class FilterNestedHash < Filter
     Plugin.register_filter("nested_hash", self)
 
+    config_param :connector, :string, :default => nil
+
     def configure conf
       super
     end
@@ -17,7 +19,7 @@ module Fluent
     end
 
     def filter tag, time, record
-      Fluent::NestedHashFilter::NestedObject.convert record
+      Fluent::NestedHashFilter::NestedObject.convert record, connector: @connector
     end
   end
 end

data/lib/fluent/plugin/nested_hash_filter/nested_object.rb

@@ -1,21 +1,24 @@
 module Fluent
   module NestedHashFilter
-    module NestedObject
-      CONNECTOR = "."
+    class NestedObject
+      DEFAULT_CONNECTOR = "."
 
-      def self.init
-        @output = {}
-        @output_keys = []
+      def self.convert object, options = {}
+        converter = new options
+        converter.select object
+        converter.output
       end
 
-      def self.convert object
-        init
-        select object
+      attr_accessor :output, :output_keys
+      attr_reader   :connector
 
-        @output
+      def initialize options = {}
+        @output      = {}
+        @output_keys = []
+        @connector   = options[:connector] || DEFAULT_CONNECTOR
       end
 
-      def self.select object
+      def select object
         case object
         when Hash
           convert_hash object
@@ -26,25 +29,25 @@ module Fluent
         end
       end
 
-      def self.add_key key
+      def add_key key
         @output_keys.push key
       end
 
-      def self.pop_key
+      def pop_key
         @output_keys.pop
       end
 
-      def self.current_key
-        @output_keys.join CONNECTOR
+      def current_key
+        @output_keys.join connector
       end
 
-      def self.update value
+      def update value
         unless current_key.empty?
           @output.update current_key => value
         end
       end
 
-      def self.convert_hash hash
+      def convert_hash hash
         if hash.keys.empty?
           update nil
         end
@@ -56,7 +59,7 @@ module Fluent
         end
       end
 
-      def self.convert_array array
+      def convert_array array
         if array.empty?
           update nil
         end

data/lib/fluent/plugin/out_nested_hash.rb

@@ -5,6 +5,7 @@ module Fluent
     Plugin.register_output("nested_hash", self)
 
     config_param :tag_prefix, :string
+    config_param :connector,  :string, :default => nil
 
     def configure conf
       super
@@ -20,7 +21,7 @@ module Fluent
 
     def emit tag, es, chain
       es.each do |time, record|
-        record = Fluent::NestedHashFilter::NestedObject.convert record
+        record = Fluent::NestedHashFilter::NestedObject.convert record, connector: @connector
         Fluent::Engine.emit @tag_prefix + tag, time, record
       end
 

data/test/fluent/plugin/filter_nested_hash_test.rb

@@ -33,9 +33,13 @@ class FilterNestedHashTest < Test::Unit::TestCase
 
   sub_test_case "configure" do
     test "check default" do
-      assert_nothing_raised do
-        create_driver "default"
-      end
+      driver = create_driver "default"
+      assert_equal nil, driver.instance.connector
+    end
+
+    test "with connector" do
+      driver = create_driver "connector -"
+      assert_equal "-", driver.instance.connector
     end
   end
 
@@ -55,6 +59,21 @@ class FilterNestedHashTest < Test::Unit::TestCase
       assert_equal expect_message, result[:message]
     end
 
+    test "with connector" do
+      driver = emit "connector -", [
+        {a: 1, b: {c: 2, d: {e: 3, f:4}, g: [10, 20, 30]}, h: [], i: {}},
+        {a: {b: {c: 1, d: {e: 2, f:3}, g: [10, 20, 30]}, h: [], i: {}}}
+      ]
+
+      expect_message = {"a" => 1, "b-c" => 2, "b-d-e" => 3, "b-d-f" => 4, "b-g-0" => 10, "b-g-1" => 20, "b-g-2" => 30, "h" => nil, "i" => nil}
+      result = filtered driver, 0
+      assert_equal expect_message, result[:message]
+
+      expect_message = {"a-b-c" => 1, "a-b-d-e" => 2, "a-b-d-f" => 3, "a-b-g-0" => 10, "a-b-g-1" => 20, "a-b-g-2" => 30, "a-h" => nil, "a-i" => nil}
+      result = filtered driver, 1
+      assert_equal expect_message, result[:message]
+    end
+
     test "with invalid record" do
       driver = emit "", ["message", {hoge: 1, fuga: {"test0" => 2, "test1" => 3}}]
 

data/test/fluent/plugin/nested_hash_filter/nested_object_test.rb

@@ -3,12 +3,12 @@ require "fluent/plugin/nested_hash_filter/nested_object"
 
 class NestedObjectTest < Test::Unit::TestCase
   setup do
-    klass.init
+    @instance = klass.new
   end
 
   sub_test_case "constants" do
-    test "::CONNECTOR" do
-      assert_equal ".", klass::CONNECTOR
+    test "::DEFAULT_CONNECTOR" do
+      assert_equal ".", klass::DEFAULT_CONNECTOR
     end
   end
 
@@ -35,11 +35,26 @@ class NestedObjectTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "initialize" do
+    test "without options" do
+      assert_equal Hash.new, @instance.output 
+      assert_equal Array.new, @instance.output_keys
+      assert_equal ".", @instance.connector
+    end
+
+    test "with options" do
+      @instance = klass.new connector: "-"
+      assert_equal Hash.new, @instance.output 
+      assert_equal Array.new, @instance.output_keys
+      assert_equal "-", @instance.connector
+    end
+  end
+
   sub_test_case "select" do
     test "object is hash" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = {hoge: 1, fuga: "2"}
-      klass.select object
+      @instance.select object
 
       assert_equal Set["test.hoge", "test.fuga"], output.keys.to_set
       assert_equal 1,   output["test.hoge"]
@@ -47,9 +62,9 @@ class NestedObjectTest < Test::Unit::TestCase
     end
 
     test "object is array" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = [:hoge, "fuga"]
-      klass.select object
+      @instance.select object
 
       assert_equal Set["test.0", "test.1"], output.keys.to_set
       assert_equal :hoge,  output["test.0"]
@@ -57,27 +72,27 @@ class NestedObjectTest < Test::Unit::TestCase
     end
 
     test "object is string" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = "hoge"
-      klass.select object
+      @instance.select object
 
       assert_equal Set["test"], output.keys.to_set
       assert_equal "hoge", output["test"]
     end
 
     test "object is number" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = 12345
-      klass.select object
+      @instance.select object
 
       assert_equal Set["test"], output.keys.to_set
       assert_equal 12345, output["test"]
     end
 
     test "object is nil" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = nil
-      klass.select object
+      @instance.select object
 
       assert_equal Set["test"], output.keys.to_set
       assert_equal nil, output["test"]
@@ -86,64 +101,71 @@ class NestedObjectTest < Test::Unit::TestCase
 
   sub_test_case "add_key" do
     test "add" do
-      klass.add_key "test"
+      @instance.add_key "test"
       assert_equal ["test"], output_keys
-      klass.add_key 1
+      @instance.add_key 1
       assert_equal ["test", 1], output_keys
-      klass.add_key "hoge"
+      @instance.add_key "hoge"
       assert_equal ["test", 1, "hoge"], output_keys
     end
   end
 
   sub_test_case "pop_key" do
     test "remove key by pop" do
-      klass.add_key "test"
-      klass.add_key 1
-      klass.add_key "hoge"
+      @instance.add_key "test"
+      @instance.add_key 1
+      @instance.add_key "hoge"
       assert_equal ["test", 1, "hoge"], output_keys
 
-      klass.pop_key
+      @instance.pop_key
       assert_equal ["test", 1], output_keys
 
-      klass.pop_key
+      @instance.pop_key
       assert_equal ["test"], output_keys
 
-      klass.pop_key
+      @instance.pop_key
       assert_equal [], output_keys
     end
 
     test "without output_keys" do
       assert_nothing_raised do
-        klass.pop_key
+        @instance.pop_key
       end
     end
   end
 
   sub_test_case "current_key" do
     test "with output_keys" do
-      klass.add_key "test"
-      klass.add_key 1
-      assert_equal "test.1", klass.current_key
+      @instance.add_key "test"
+      @instance.add_key 1
+      assert_equal "test.1", @instance.current_key
     end
 
     test "without output_keys" do
-      assert_equal "", klass.current_key
+      assert_equal "", @instance.current_key
+    end
+
+    test "for not default connector" do
+      @instance = klass.new connector: "-"
+      @instance.add_key "test"
+      @instance.add_key 1
+      assert_equal "test-1", @instance.current_key
     end
   end
 
   sub_test_case "update" do
     test "with current_key" do
-      klass.add_key "test"
-      klass.add_key 1
-      klass.update 12345
+      @instance.add_key "test"
+      @instance.add_key 1
+      @instance.update 12345
 
       assert_equal ["test.1"], output.keys
       assert_equal 12345, output["test.1"]
     end
 
     test "without current_key" do
-      assert klass.current_key.empty?
-      klass.update 12345
+      assert @instance.current_key.empty?
+      @instance.update 12345
 
       assert_equal [], output.keys
     end
@@ -151,17 +173,17 @@ class NestedObjectTest < Test::Unit::TestCase
 
   sub_test_case "convert_hash" do
     test "hash is blank" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = {}
-      klass.convert_hash object
+      @instance.convert_hash object
       assert_equal ["test"], output.keys
       assert_equal nil, output["test"]
     end
 
     test "convert" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = {"a" => "hoge", "b" => 1, "c" => {"hoge" => "fuga"}, "d" => [10, 20], "e" => nil}
-      klass.convert_hash object
+      @instance.convert_hash object
       assert_equal ["test.a", "test.b", "test.c.hoge", "test.d.0", "test.d.1", "test.e"], output.keys
       assert_equal "hoge", output["test.a"]
       assert_equal 1,      output["test.b"]
@@ -174,17 +196,17 @@ class NestedObjectTest < Test::Unit::TestCase
 
   sub_test_case "convert_array" do
     test "array is empty" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = []
-      klass.convert_array object
+      @instance.convert_array object
       assert_equal ["test"], output.keys
       assert_equal nil, output["test"]
     end
 
     test "convert" do
-      klass.add_key "test"
+      @instance.add_key "test"
       object = ["hoge", 1, {"hoge" => "fuga"}, [10, 20], nil]
-      klass.convert_array object
+      @instance.convert_array object
       assert_equal ["test.0", "test.1", "test.2.hoge", "test.3.0", "test.3.1", "test.4"], output.keys
       assert_equal "hoge", output["test.0"]
       assert_equal 1,      output["test.1"]
@@ -196,11 +218,11 @@ class NestedObjectTest < Test::Unit::TestCase
   end
 
   def output
-    klass.instance_variable_get :@output
+    @instance.output
   end
 
   def output_keys
-    klass.instance_variable_get :@output_keys
+    @instance.output_keys
   end
 
   def klass

data/test/fluent/plugin/out_nested_hash_test.rb

@@ -33,13 +33,20 @@ class OutNestedHashTest < Test::Unit::TestCase
 
   sub_test_case "configure" do
     test "check_default" do
-      create_driver "tag_prefix default"
+      driver = create_driver "tag_prefix default"
+      assert_equal "default", driver.instance.tag_prefix
+      assert_equal nil, driver.instance.connector
     end
 
-    test "tag_prefix" do
+    test "with tag_prefix" do
       driver = create_driver "tag_prefix filtered."
       assert_equal "filtered.", driver.instance.tag_prefix
     end
+
+    test "with connector" do
+      driver = create_driver "tag_prefix default\nconnector -"
+      assert_equal "-", driver.instance.connector
+    end
   end
 
   sub_test_case "emit" do
@@ -60,6 +67,23 @@ class OutNestedHashTest < Test::Unit::TestCase
       assert_equal expect_message, result[:message]
     end
 
+    test "with connector" do
+      driver = emit "tag_prefix filtered.\nconnector -", "test", [
+        {a: 1, b: {c: 2, d: {e: 3, f:4}, g: [10, 20, 30]}, h: [], i: {}},
+        {a: {b: {c: 1, d: {e: 2, f:3}, g: [10, 20, 30]}, h: [], i: {}}}
+      ]
+
+      expect_message = {"a" => 1, "b-c" => 2, "b-d-e" => 3, "b-d-f" => 4, "b-g-0" => 10, "b-g-1" => 20, "b-g-2" => 30, "h" => nil, "i" => nil}
+      result = emitted driver, 0
+      assert_equal "filtered.test", result[:tag]
+      assert_equal expect_message, result[:message]
+
+      expect_message = {"a-b-c" => 1, "a-b-d-e" => 2, "a-b-d-f" => 3, "a-b-g-0" => 10, "a-b-g-1" => 20, "a-b-g-2" => 30, "a-h" => nil, "a-i" => nil}
+      result = emitted driver, 1
+      assert_equal "filtered.test", result[:tag]
+      assert_equal expect_message, result[:message]
+    end
+
     test "with invalid record" do
       driver = emit "tag_prefix filtered.", "test", ["message", {hoge: 1, fuga: {"test0" => 2, "test1" => 3}}]
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-nested-hash-filter
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors:
 - sugilog
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-28 00:00:00.000000000 Z
+date: 2015-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler