fluent-plugin-nested-hash-filter 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8086358c385e65ba72177e5d387af2dbc59d9327
-  data.tar.gz: 00dd2d49d61abda94163664ebcc5147dc4a8439c
+  metadata.gz: c092a833fd6982c036035b9292b70b08a25c0ae8
+  data.tar.gz: 0c9291bb5874ed145888843d59e1e455cda1676c
 SHA512:
-  metadata.gz: 9bf597fb8200cb91327b6e061b9749ad88f17a8af45037ff7360eef64d80acbcf3c7fd7dd65c1f5f2f8814d269e37eeb3e83f5ef5f27e9b633b4a88bd412d676
-  data.tar.gz: 9e850518c9d05ea283c89d473269301b0923dc275dc60a74f671841aa1c859f545509bbe9a42c059a16628e8a68e991359869edadec2f5dcb81bafd055a65ff4
+  metadata.gz: 1a94d9e631ce5a6a85470ac71bf48dcda611b1303f262a620b0277565b106086b364346e956ad051b388426f9212b93477af096a65690e388bf1a9cf60be68d2
+  data.tar.gz: 0842ec1d2f922a108576476d1499ab4317cc07039e34ab7e67b17044a52397aaee7b2b5c1e73cf3b8b956fb82225bcc6922cbc970dba3cef7d5e7cd09f887879

data/README.md

@@ -26,6 +26,7 @@ Add config to your `td-agent.conf`
 <filter {MATCH_PATTERN}>
   type nested_hash
   connector .
+  acts_as_json ["key1", "key2-1.key2-2"]
 </filter>
 ```
 
@@ -36,6 +37,7 @@ Add config to your `td-agent.conf`
   type       nested_hash
   tag_prefix {PREFIX}
   connector  .
+  acts_as_json ["key1", "key2-1.key2-2"]
 </match>
 ```
 
@@ -43,6 +45,8 @@ Add config to your `td-agent.conf`
 
 `connector` is optional parameter to connect nested-keys. (default: `.`) not for tag prefix connection.
 
+`acts_as_json` is optional parameter to convert string to hash in json format. (default: `[]` empty)
+
 - ex: matched tag is `access.log` and `tag_prefix` is `converted.`, then log will be passed with tag name `converted.access.log`.
 
 ## Development
@@ -51,6 +55,13 @@ After checking out the repo, run `bundle install` to install dependencies.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Demo script
+
+```
+bundle exec ruby script/demo_logger.rb
+```
+
+
 ## Contributing
 
 1. Fork it ( https://github.com/sugilog/fluent-plugin-nested-hash-filter/fork )

data/fluent-plugin-nested-hash-filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-nested-hash-filter"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["sugilog"]
   spec.email         = ["sugilog@gmail.com"]
 
@@ -21,4 +21,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit"
   spec.add_development_dependency "guard"
   spec.add_development_dependency "guard-test"
+  spec.add_development_dependency "faker"
 end

data/lib/fluent/plugin/filter_nested_hash.rb

@@ -5,6 +5,7 @@ module Fluent
     Plugin.register_filter("nested_hash", self)
 
     config_param :connector, :string, :default => nil
+    config_param :acts_as_json, :array, :default => []
 
     def configure conf
       super
@@ -19,7 +20,7 @@ module Fluent
     end
 
     def filter tag, time, record
-      Fluent::NestedHashFilter::NestedObject.convert record, connector: @connector
+      Fluent::NestedHashFilter::NestedObject.convert record, connector: @connector, jsonify_keys: @acts_as_json
     end
   end
 end

data/lib/fluent/plugin/nested_hash_filter/nested_object.rb

@@ -1,3 +1,5 @@
+require 'json'
+
 module Fluent
   module NestedHashFilter
     class NestedObject
@@ -9,13 +11,14 @@ module Fluent
         converter.output
       end
 
-      attr_accessor :output, :output_keys
+      attr_accessor :output, :output_keys, :jsonify_keys
       attr_reader   :connector
 
       def initialize options = {}
-        @output      = {}
-        @output_keys = []
-        @connector   = options[:connector] || DEFAULT_CONNECTOR
+        @output       = {}
+        @output_keys  = []
+        @connector    = options[:connector] || DEFAULT_CONNECTOR
+        @jsonify_keys = init_jsonify_keys options[:jsonify_keys]
       end
 
       def select object
@@ -41,10 +44,45 @@ module Fluent
         @output_keys.join connector
       end
 
+      def acts_as_json value
+        json = JSON.parse value
+        select json
+      rescue TypeError => error
+        err = JSON::ParserError.new error.message
+        err.set_backtrace err.backtrace
+        raise err
+      rescue JSON::ParserError => error
+        raise error
+      ensure
+        jsonified!
+      end
+
+      def jsonify_key?
+        !!@jsonify_keys[current_key]
+      end
+
+      def jsonified!
+        @jsonify_keys[current_key] = false
+      end
+
+      def init_jsonify_keys keys
+        keys = keys || []
+        values = [true] * keys.size
+        zipped = keys.zip values
+        Hash[ zipped ]
+      end
+
       def update value
-        unless current_key.empty?
+        case
+        when current_key.empty?
+          return
+        when jsonify_key?
+          acts_as_json value
+        else
           @output.update current_key => value
         end
+      rescue JSON::ParserError
+        @output.update current_key => value
       end
 
       def convert_hash hash

data/lib/fluent/plugin/out_nested_hash.rb

@@ -6,6 +6,7 @@ module Fluent
 
     config_param :tag_prefix, :string
     config_param :connector,  :string, :default => nil
+    config_param :acts_as_json, :array, :default => []
 
     def configure conf
       super
@@ -21,7 +22,7 @@ module Fluent
 
     def emit tag, es, chain
       es.each do |time, record|
-        record = Fluent::NestedHashFilter::NestedObject.convert record, connector: @connector
+        record = Fluent::NestedHashFilter::NestedObject.convert record, connector: @connector, jsonify_keys: @acts_as_json
         Fluent::Engine.emit @tag_prefix + tag, time, record
       end
 

data/script/demo_logger.rb

@@ -0,0 +1,72 @@
+require "rubygems"
+require "logger"
+require "json"
+require "faker"
+require "uri"
+
+FILE_PATH = "/tmp/fluent-plugin-nested-hash-filter.demo.log"
+@count = 0
+
+def main
+  loop do
+    object = generate
+    output object
+    sleep 0.1
+    @count += 1
+  end
+end
+
+def generate
+  case @count % 100
+  when 0..85
+    session_id = @count
+  when 86..95
+    session_id = ""
+  when 96..98
+    session_id = @count.to_s.rjust 10, "0"
+  else
+    session_id = Faker::Lorem.characters 20
+  end
+
+  case @count % 100
+  when 0..98
+    access = Time.now.to_s
+    number = Faker::Number.number 10
+  else
+    access = "undefined"
+    number = ""
+  end
+
+  uri = URI.parse Faker::Internet.url
+  expired = Time.now + ( 3 * 60 * 60 * 24 )
+  adsid = Faker::Number.number 8
+
+  {
+    session_id: session_id,
+    session:    {
+      expired: expired,
+      ads:     {
+        _id:  adsid,
+        list: Faker::Lorem.words,
+      }
+    }.to_json,
+    count:      number,
+    access:     access,
+    uri:        uri.path,
+    query: {
+      keyword:  Faker::Lorem.words,
+      page:     0
+    },
+    response:   Faker::Lorem.sentence,
+    email:      Faker::Internet.safe_email
+  }
+end
+
+def output object
+  File.open FILE_PATH, "a" do |file|
+    line = object.to_json
+    file.puts line
+  end
+end
+
+main

data/test/fluent/plugin/filter_nested_hash_test.rb

@@ -35,11 +35,19 @@ class FilterNestedHashTest < Test::Unit::TestCase
     test "check default" do
       driver = create_driver "default"
       assert_equal nil, driver.instance.connector
+      assert_equal [], driver.instance.acts_as_json
     end
 
     test "with connector" do
       driver = create_driver "connector -"
       assert_equal "-", driver.instance.connector
+      assert_equal [], driver.instance.acts_as_json
+    end
+
+    test "with acts_as_json" do
+      driver = create_driver "acts_as_json [\"hoge.1\", \"fuga\"]"
+      assert_equal nil, driver.instance.connector
+      assert_equal ["hoge.1", "fuga"], driver.instance.acts_as_json
     end
   end
 
@@ -74,6 +82,16 @@ class FilterNestedHashTest < Test::Unit::TestCase
       assert_equal expect_message, result[:message]
     end
 
+    test "with acts_as_json" do
+      driver = emit "acts_as_json [\"a\",\"b.c\"]", [
+        {a: "[100, 200]", b: {c: '{"x": 1, "y": 2, "z": 3}', d: {e: 3, f:4}, g: [10, 20, 30]}, h: [], i: {}},
+      ]
+
+      expect_message = {"a.0" => 100, "a.1" => 200, "b.c.x" => 1, "b.c.y" => 2, "b.c.z" => 3, "b.d.e" => 3, "b.d.f" => 4, "b.g.0" => 10, "b.g.1" => 20, "b.g.2" => 30, "h" => nil, "i" => nil}
+      result = filtered driver, 0
+      assert_equal expect_message, result[:message]
+    end
+
     test "with invalid record" do
       driver = emit "", ["message", {hoge: 1, fuga: {"test0" => 2, "test1" => 3}}]
 

data/test/fluent/plugin/nested_hash_filter/nested_object_test.rb

@@ -40,13 +40,25 @@ class NestedObjectTest < Test::Unit::TestCase
       assert_equal Hash.new, @instance.output 
       assert_equal Array.new, @instance.output_keys
       assert_equal ".", @instance.connector
+      assert_equal Hash.new, @instance.jsonify_keys
     end
 
-    test "with options" do
+    test "with connector option" do
       @instance = klass.new connector: "-"
       assert_equal Hash.new, @instance.output 
       assert_equal Array.new, @instance.output_keys
       assert_equal "-", @instance.connector
+      assert_equal Hash.new, @instance.jsonify_keys
+    end
+
+    test "with jsonify_keys option" do
+      @instance = klass.new jsonify_keys: ["hoge.1", "fuga"]
+      assert_equal Hash.new, @instance.output 
+      assert_equal Array.new, @instance.output_keys
+      assert_equal ".", @instance.connector
+      assert_equal ["fuga", "hoge.1"], @instance.jsonify_keys.keys.sort
+      assert @instance.jsonify_keys["hoge.1"]
+      assert @instance.jsonify_keys["fuga"]
     end
   end
 
@@ -153,6 +165,129 @@ class NestedObjectTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "acts_as_json" do
+    test "with strinfied json" do
+      value = {"hoge" => [ 10, 20 ], "fuga" => Date.new(2015, 1, 23)}.to_json
+      assert_instance_of String, value
+      result = @instance.acts_as_json value
+      assert_equal Set["fuga", "hoge"], result.keys.to_set
+      assert_equal [10, 20], result["hoge"]
+      assert_equal Date.new(2015, 1, 23).to_s, result["fuga"]
+    end
+
+    test "with non json format string" do
+      value = "hoge: [10, 20], fuga: 'undefined'"
+      result = nil
+
+      assert_raise JSON::ParserError do
+        result = @instance.acts_as_json value
+      end
+
+      assert_nil result
+    end
+
+    test "with nil" do
+      value = nil
+      result = nil
+
+      assert_raise JSON::ParserError do
+        result = @instance.acts_as_json value
+      end
+
+      assert_nil result
+    end
+
+    test "with integer" do
+      value = 123
+      result = nil
+
+      assert_raise JSON::ParserError do
+        result = @instance.acts_as_json value
+      end
+
+      assert_nil result
+    end
+
+    test "with hash" do
+      value = {"hoge" => [ 10, 20 ], "fuga" => Date.new(2015, 1, 23)}
+      result = nil
+
+      assert_raise JSON::ParserError do
+        result = @instance.acts_as_json value
+      end
+
+      assert_nil result
+    end
+
+    test "with array" do
+      value = [ 10, 20 ]
+      result = nil
+
+      assert_raise JSON::ParserError do
+        result = @instance.acts_as_json value
+      end
+
+      assert_nil result
+    end
+  end
+
+  sub_test_case "jsonify_keys?" do
+    test "with no key" do
+      @instance.jsonify_keys = {}
+      @instance.output_keys = ["hoge", 0]
+      assert !@instance.jsonify_key?
+    end
+
+    test "with key" do
+      @instance.jsonify_keys = { "hoge.1" => true, "fuga" => false }
+
+      @instance.output_keys = ["hoge", 1]
+      assert @instance.jsonify_key?
+
+      @instance.output_keys = ["fuga"]
+      assert !@instance.jsonify_key?
+    end
+  end
+
+  sub_test_case "jsonified!" do
+    test "set false" do
+      @instance.jsonify_keys = { "hoge.1" => true, "fuga" => false }
+
+      @instance.output_keys = ["hoge", 1]
+      @instance.jsonified!
+      assert !@instance.jsonify_keys["hoge.1"]
+
+      @instance.output_keys = ["fuga"]
+      @instance.jsonified!
+      assert !@instance.jsonify_keys["fuga"]
+
+      @instance.output_keys = ["piyo"]
+      @instance.jsonified!
+      assert !@instance.jsonify_keys["piyo"]
+    end
+  end
+
+  sub_test_case "init_jsonify_keys" do
+    test "with array keys" do
+      jsonify_keys = @instance.init_jsonify_keys ["hoge.1", "fuga"]
+      assert_equal Set["fuga", "hoge.1"], jsonify_keys.keys.to_set
+      assert jsonify_keys["hoge.1"]
+      assert jsonify_keys["fuga"]
+    end
+
+    test "with empty keys" do
+      jsonify_keys = @instance.init_jsonify_keys []
+      assert jsonify_keys.keys.empty?
+      assert_equal Hash.new, jsonify_keys
+    end
+
+    test "with nil" do
+      jsonify_keys = @instance.init_jsonify_keys nil
+      assert jsonify_keys.keys.empty?
+      assert_equal Hash.new, jsonify_keys
+    end
+  end
+
   sub_test_case "update" do
     test "with current_key" do
       @instance.add_key "test"
@@ -169,6 +304,39 @@ class NestedObjectTest < Test::Unit::TestCase
 
       assert_equal [], output.keys
     end
+
+    test "with jsonify_keys" do
+      @instance.jsonify_keys = @instance.init_jsonify_keys ["hoge.1", "fuga"]
+      value = {"test" => [12345, 6789], "piyo" => nil}.to_json
+
+      @instance.add_key "hoge"
+      @instance.add_key 0
+      @instance.update value
+      assert_equal ["hoge.0"], output.keys
+      assert_equal value, output["hoge.0"]
+
+      @instance.pop_key
+      @instance.pop_key
+      @instance.add_key "hoge"
+      @instance.add_key 1
+      @instance.update value
+      assert_equal Set["hoge.0", "hoge.1.piyo", "hoge.1.test.0", "hoge.1.test.1"], output.keys.to_set
+      assert_equal value, output["hoge.0"]
+      assert_equal 12345, output["hoge.1.test.0"]
+      assert_equal 6789, output["hoge.1.test.1"]
+      assert_equal nil, output["hoge.1.piyo"]
+
+      @instance.pop_key
+      @instance.pop_key
+      @instance.add_key "fuga"
+      @instance.update 987654321
+      assert_equal Set["hoge.0", "hoge.1.piyo", "hoge.1.test.0", "hoge.1.test.1", "fuga"], output.keys.to_set
+      assert_equal value, output["hoge.0"]
+      assert_equal 12345, output["hoge.1.test.0"]
+      assert_equal 6789, output["hoge.1.test.1"]
+      assert_equal nil, output["hoge.1.piyo"]
+      assert_equal 987654321, output["fuga"]
+    end
   end
 
   sub_test_case "convert_hash" do
@@ -184,7 +352,7 @@ class NestedObjectTest < Test::Unit::TestCase
       @instance.add_key "test"
       object = {"a" => "hoge", "b" => 1, "c" => {"hoge" => "fuga"}, "d" => [10, 20], "e" => nil}
       @instance.convert_hash object
-      assert_equal ["test.a", "test.b", "test.c.hoge", "test.d.0", "test.d.1", "test.e"], output.keys
+      assert_equal Set["test.a", "test.b", "test.c.hoge", "test.d.0", "test.d.1", "test.e"], output.keys.to_set
       assert_equal "hoge", output["test.a"]
       assert_equal 1,      output["test.b"]
       assert_equal "fuga", output["test.c.hoge"]
@@ -207,7 +375,7 @@ class NestedObjectTest < Test::Unit::TestCase
       @instance.add_key "test"
       object = ["hoge", 1, {"hoge" => "fuga"}, [10, 20], nil]
       @instance.convert_array object
-      assert_equal ["test.0", "test.1", "test.2.hoge", "test.3.0", "test.3.1", "test.4"], output.keys
+      assert_equal Set["test.0", "test.1", "test.2.hoge", "test.3.0", "test.3.1", "test.4"], output.keys.to_set
       assert_equal "hoge", output["test.0"]
       assert_equal 1,      output["test.1"]
       assert_equal "fuga", output["test.2.hoge"]

data/test/fluent/plugin/out_nested_hash_test.rb

@@ -36,16 +36,27 @@ class OutNestedHashTest < Test::Unit::TestCase
       driver = create_driver "tag_prefix default"
       assert_equal "default", driver.instance.tag_prefix
       assert_equal nil, driver.instance.connector
+      assert_equal [], driver.instance.acts_as_json
     end
 
     test "with tag_prefix" do
       driver = create_driver "tag_prefix filtered."
       assert_equal "filtered.", driver.instance.tag_prefix
+      assert_equal [], driver.instance.acts_as_json
     end
 
     test "with connector" do
       driver = create_driver "tag_prefix default\nconnector -"
+      assert_equal "default", driver.instance.tag_prefix
       assert_equal "-", driver.instance.connector
+      assert_equal [], driver.instance.acts_as_json
+    end
+
+    test "with acts_as_json" do
+      driver = create_driver "acts_as_json [\"hoge.1\", \"fuga\"]\ntag_prefix default"
+      assert_equal "default", driver.instance.tag_prefix
+      assert_equal nil, driver.instance.connector
+      assert_equal ["hoge.1", "fuga"], driver.instance.acts_as_json
     end
   end
 
@@ -84,6 +95,17 @@ class OutNestedHashTest < Test::Unit::TestCase
       assert_equal expect_message, result[:message]
     end
 
+    test "with acts_as_json" do
+      driver = emit "acts_as_json [\"a\",\"b.c\"]\ntag_prefix filtered.", "test", [
+        {a: "[100, 200]", b: {c: '{"x": 1, "y": 2, "z": 3}', d: {e: 3, f:4}, g: [10, 20, 30]}, h: [], i: {}},
+      ]
+
+      expect_message = {"a.0" => 100, "a.1" => 200, "b.c.x" => 1, "b.c.y" => 2, "b.c.z" => 3, "b.d.e" => 3, "b.d.f" => 4, "b.g.0" => 10, "b.g.1" => 20, "b.g.2" => 30, "h" => nil, "i" => nil}
+      result = emitted driver, 0
+      assert_equal "filtered.test", result[:tag]
+      assert_equal expect_message, result[:message]
+    end
+
     test "with invalid record" do
       driver = emit "tag_prefix filtered.", "test", ["message", {hoge: 1, fuga: {"test0" => 2, "test1" => 3}}]
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-nested-hash-filter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - sugilog
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-04 00:00:00.000000000 Z
+date: 2015-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fluent Plugin for converting nested hash into flatten key-value pair.
 email:
 - sugilog@gmail.com
@@ -111,6 +125,7 @@ files:
 - lib/fluent/plugin/filter_nested_hash.rb
 - lib/fluent/plugin/nested_hash_filter/nested_object.rb
 - lib/fluent/plugin/out_nested_hash.rb
+- script/demo_logger.rb
 - test/fluent/plugin/filter_nested_hash_test.rb
 - test/fluent/plugin/nested_hash_filter/nested_object_test.rb
 - test/fluent/plugin/out_nested_hash_test.rb