fluent-plugin-nested-hash-filter 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 64f83a066f2e3c8d2a3453f62615abc36a1d664a
+  data.tar.gz: 5da7b8ef0de45bf917245a3eb5fac1867b33d659
+SHA512:
+  metadata.gz: 2f009f27550f02a2f6333ef8ffca7d8925ccc6b45e3763abc2827af2fa4cd440cd87b273cf6359739d87f11a097d29c61b02e452313d74cb880231d6fcdda651
+  data.tar.gz: 0dde134af29ad68a084ea62431b12ac2bef0c2c972a228cbd5053ace7bb549e72888a3d232f83cdebb3dec337880e0c57ccba2c44b892e7c8d95e3fc7b6ef08d

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor/

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 2.2.2
+  - 2.1
+  - 2.0.0
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-nested-hash-filter.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,5 @@
+guard :test do
+  watch(%r{^test/.+_test\.rb$})
+  watch('test/test_helper.rb')  { 'test' }
+  watch(%r{^lib/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
+end

data/README.md

@@ -0,0 +1,42 @@
+# fluent-plugin-nested-hash-filter
+
+[![Build Status](https://travis-ci.org/sugilog/fluent-plugin-nested-hash-filter.png?branch=master)](https://travis-ci.org/sugilog/fluent-plugin-nested-hash-filter.svg)
+
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fluent/plugin/filter_nested_hash`.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-nested-hash-filter'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-nested-hash-filter
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/sugilog/fluent-plugin-nested-hash-filter/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+desc "Run tests"
+Rake::TestTask.new do |test|
+  test.name = :test
+  test.libs = ["lib", "test"]
+  test.test_files = FileList["test/**/*_test.rb"]
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-nested-hash-filter.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-nested-hash-filter"
+  spec.version       = "0.0.1"
+  spec.authors       = ["sugilog"]
+  spec.email         = ["sugilog@gmail.com"]
+
+  spec.summary       = %q{Fluent Plugin for converting nested hash into flatten key-value pair.}
+  spec.description   = %q{Fluent Plugin for converting nested hash into flatten key-value pair.}
+  spec.homepage      = "https://github.com/sugilog/fluent-plugin-nested-hash-filter"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency "fluentd"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-test"
+end

data/lib/fluent/plugin/filter_nested_hash.rb

@@ -0,0 +1,23 @@
+require "fluent/plugin/nested_hash_filter/nested_object"
+
+module Fluent
+  class FilterNestedHash < Filter
+    Plugin.register_filter("nested_hash_filter", self)
+
+    def configure conf
+      super
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def filter tag, time, record
+      Fluent::NestedHashFilter::NestedObject.convert record
+    end
+  end
+end

data/lib/fluent/plugin/nested_hash_filter/nested_object.rb

@@ -0,0 +1,72 @@
+module Fluent
+  module NestedHashFilter
+    module NestedObject
+      CONNECTOR = "."
+
+      def self.init
+        @output = {}
+        @output_keys = []
+      end
+
+      def self.convert object
+        init
+        select object
+
+        @output
+      end
+
+      def self.select object
+        case object
+        when Hash
+          convert_hash object
+        when Array
+          convert_array object
+        else
+          update object
+        end
+      end
+
+      def self.add_key key
+        @output_keys.push key
+      end
+
+      def self.pop_key
+        @output_keys.pop
+      end
+
+      def self.current_key
+        @output_keys.join CONNECTOR
+      end
+
+      def self.update value
+        unless current_key.empty?
+          @output.update current_key => value
+        end
+      end
+
+      def self.convert_hash hash
+        if hash.keys.empty?
+          update nil
+        end
+
+        hash.each do |key, value|
+          add_key key
+          select  value
+          pop_key
+        end
+      end
+
+      def self.convert_array array
+        if array.empty?
+          update nil
+        end
+
+        array.each_with_index do |value, index|
+          add_key index
+          select  value
+          pop_key
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_nested_hash.rb

@@ -0,0 +1,28 @@
+require "fluent/plugin/nested_hash_filter/nested_object"
+
+module Fluent
+  class OutNestedHash < Output
+    Plugin.register_output("nested_hash_filter", self)
+
+    def configure conf
+      super
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def emit tag, es, chain
+      es.each do |time, record|
+        record = Fluent::NestedHashFilter::NestedObject.convert record
+        Fluent::Engine.emit tag, time, record
+      end
+
+      chain.next
+    end
+  end
+end

data/test/fluent/plugin/filter_nested_hash_test.rb

@@ -0,0 +1,70 @@
+require "test_helper"
+require "fluent/plugin/filter_nested_hash"
+
+class FilterNestedHashTest < Test::Unit::TestCase
+  include Fluent
+
+  setup do
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  def create_driver config = ""
+    driver = Test::FilterTestDriver.new FilterNestedHash
+    driver.configure config, true
+  end
+
+  def emit config, messages
+    driver = create_driver config
+
+    driver.run do
+      messages.each do |message|
+        driver.emit message, @time
+      end
+    end
+    
+    driver
+  end
+
+  def filtered driver, index
+    result = driver.filtered_as_array[index]
+    {tag: result[0], time: result[1], message: result[2]}
+  end
+
+  sub_test_case "configure" do
+    test "check default" do
+      assert_nothing_raised do
+        create_driver "default"
+      end
+    end
+  end
+
+  sub_test_case "filter" do
+    test "with valid record" do
+      driver = emit "", [
+        {a: 1, b: {c: 2, d: {e: 3, f:4}, g: [10, 20, 30]}, h: [], i: {}},
+        {a: {b: {c: 1, d: {e: 2, f:3}, g: [10, 20, 30]}, h: [], i: {}}}
+      ]
+
+      expect_message = {"a" => 1, "b.c" => 2, "b.d.e" => 3, "b.d.f" => 4, "b.g.0" => 10, "b.g.1" => 20, "b.g.2" => 30, "h" => nil, "i" => nil}
+      result = filtered driver, 0
+      assert_equal expect_message, result[:message]
+
+      expect_message = {"a.b.c" => 1, "a.b.d.e" => 2, "a.b.d.f" => 3, "a.b.g.0" => 10, "a.b.g.1" => 20, "a.b.g.2" => 30, "a.h" => nil, "a.i" => nil}
+      result = filtered driver, 1
+      assert_equal expect_message, result[:message]
+    end
+
+    test "with invalid record" do
+      driver = emit "", ["message", {hoge: 1, fuga: {"test0" => 2, "test1" => 3}}]
+
+      expect_message = {}
+      result = filtered driver, 0
+      assert_equal expect_message, result[:message]
+
+      expect_message = {"hoge" => 1, "fuga.test0" => 2, "fuga.test1" => 3}
+      result = filtered driver, 1
+      assert_equal expect_message, result[:message]
+    end
+  end
+end

data/test/fluent/plugin/nested_hash_filter/nested_object_test.rb

@@ -0,0 +1,209 @@
+require "test_helper"
+require "fluent/plugin/nested_hash_filter/nested_object"
+
+class NestedObjectTest < Test::Unit::TestCase
+  setup do
+    klass.init
+  end
+
+  sub_test_case "constants" do
+    test "::CONNECTOR" do
+      assert_equal ".", klass::CONNECTOR
+    end
+  end
+
+  sub_test_case "convert" do
+    test "convert by multi time" do
+      hash = {"a" => 1, "b" => {"c" => 2, "d" => 3}, "e" => [4, 5], "f" => {}, "g" => []}
+      output = klass.convert hash
+      assert_equal Set["a", "b.c", "b.d", "e.0", "e.1", "f", "g"], output.keys.to_set
+      assert_equal 1,   output["a"]
+      assert_equal 2,   output["b.c"]
+      assert_equal 3,   output["b.d"]
+      assert_equal 4,   output["e.0"]
+      assert_equal 5,   output["e.1"]
+      assert_equal nil, output["f"]
+      assert_equal nil, output["g"]
+
+      hash = {"a" => {"b" => {"c" => 1, "d" => 2}, "e" => [3, 4]}}
+      output = klass.convert hash
+      assert_equal Set["a.b.c", "a.b.d", "a.e.0", "a.e.1"], output.keys.to_set
+      assert_equal 1, output["a.b.c"]
+      assert_equal 2, output["a.b.d"]
+      assert_equal 3, output["a.e.0"]
+      assert_equal 4, output["a.e.1"]
+    end
+  end
+
+  sub_test_case "select" do
+    test "object is hash" do
+      klass.add_key "test"
+      object = {hoge: 1, fuga: "2"}
+      klass.select object
+
+      assert_equal Set["test.hoge", "test.fuga"], output.keys.to_set
+      assert_equal 1,   output["test.hoge"]
+      assert_equal "2", output["test.fuga"]
+    end
+
+    test "object is array" do
+      klass.add_key "test"
+      object = [:hoge, "fuga"]
+      klass.select object
+
+      assert_equal Set["test.0", "test.1"], output.keys.to_set
+      assert_equal :hoge,  output["test.0"]
+      assert_equal "fuga", output["test.1"]
+    end
+
+    test "object is string" do
+      klass.add_key "test"
+      object = "hoge"
+      klass.select object
+
+      assert_equal Set["test"], output.keys.to_set
+      assert_equal "hoge", output["test"]
+    end
+
+    test "object is number" do
+      klass.add_key "test"
+      object = 12345
+      klass.select object
+
+      assert_equal Set["test"], output.keys.to_set
+      assert_equal 12345, output["test"]
+    end
+
+    test "object is nil" do
+      klass.add_key "test"
+      object = nil
+      klass.select object
+
+      assert_equal Set["test"], output.keys.to_set
+      assert_equal nil, output["test"]
+    end
+  end
+
+  sub_test_case "add_key" do
+    test "add" do
+      klass.add_key "test"
+      assert_equal ["test"], output_keys
+      klass.add_key 1
+      assert_equal ["test", 1], output_keys
+      klass.add_key "hoge"
+      assert_equal ["test", 1, "hoge"], output_keys
+    end
+  end
+
+  sub_test_case "pop_key" do
+    test "remove key by pop" do
+      klass.add_key "test"
+      klass.add_key 1
+      klass.add_key "hoge"
+      assert_equal ["test", 1, "hoge"], output_keys
+
+      klass.pop_key
+      assert_equal ["test", 1], output_keys
+
+      klass.pop_key
+      assert_equal ["test"], output_keys
+
+      klass.pop_key
+      assert_equal [], output_keys
+    end
+
+    test "without output_keys" do
+      assert_nothing_raised do
+        klass.pop_key
+      end
+    end
+  end
+
+  sub_test_case "current_key" do
+    test "with output_keys" do
+      klass.add_key "test"
+      klass.add_key 1
+      assert_equal "test.1", klass.current_key
+    end
+
+    test "without output_keys" do
+      assert_equal "", klass.current_key
+    end
+  end
+
+  sub_test_case "update" do
+    test "with current_key" do
+      klass.add_key "test"
+      klass.add_key 1
+      klass.update 12345
+
+      assert_equal ["test.1"], output.keys
+      assert_equal 12345, output["test.1"]
+    end
+
+    test "without current_key" do
+      assert klass.current_key.empty?
+      klass.update 12345
+
+      assert_equal [], output.keys
+    end
+  end
+
+  sub_test_case "convert_hash" do
+    test "hash is blank" do
+      klass.add_key "test"
+      object = {}
+      klass.convert_hash object
+      assert_equal ["test"], output.keys
+      assert_equal nil, output["test"]
+    end
+
+    test "convert" do
+      klass.add_key "test"
+      object = {"a" => "hoge", "b" => 1, "c" => {"hoge" => "fuga"}, "d" => [10, 20], "e" => nil}
+      klass.convert_hash object
+      assert_equal ["test.a", "test.b", "test.c.hoge", "test.d.0", "test.d.1", "test.e"], output.keys
+      assert_equal "hoge", output["test.a"]
+      assert_equal 1,      output["test.b"]
+      assert_equal "fuga", output["test.c.hoge"]
+      assert_equal 10,     output["test.d.0"]
+      assert_equal 20,     output["test.d.1"]
+      assert_equal nil,    output["test.e"]
+    end
+  end
+
+  sub_test_case "convert_array" do
+    test "array is empty" do
+      klass.add_key "test"
+      object = []
+      klass.convert_array object
+      assert_equal ["test"], output.keys
+      assert_equal nil, output["test"]
+    end
+
+    test "convert" do
+      klass.add_key "test"
+      object = ["hoge", 1, {"hoge" => "fuga"}, [10, 20], nil]
+      klass.convert_array object
+      assert_equal ["test.0", "test.1", "test.2.hoge", "test.3.0", "test.3.1", "test.4"], output.keys
+      assert_equal "hoge", output["test.0"]
+      assert_equal 1,      output["test.1"]
+      assert_equal "fuga", output["test.2.hoge"]
+      assert_equal 10,     output["test.3.0"]
+      assert_equal 20,     output["test.3.1"]
+      assert_equal nil,    output["test.4"]
+    end
+  end
+
+  def output
+    klass.instance_variable_get :@output
+  end
+
+  def output_keys
+    klass.instance_variable_get :@output_keys
+  end
+
+  def klass
+    Fluent::NestedHashFilter::NestedObject
+  end
+end

data/test/fluent/plugin/out_nested_hash_test.rb

@@ -0,0 +1,71 @@
+require "test_helper"
+require "fluent/plugin/out_nested_hash"
+
+class OutNestedHashTest < Test::Unit::TestCase
+  include Fluent
+
+  setup do
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  def create_driver config = "", tag = "test"
+    driver = Fluent::Test::OutputTestDriver.new OutNestedHash, tag
+    driver.configure config
+  end
+
+  def emit config, tag, messages
+    driver = create_driver config, tag
+
+    driver.run do
+      messages.each do |message|
+        driver.emit message
+      end
+    end
+
+    driver
+  end
+
+  def emitted driver, index
+    result = driver.emits[index]
+    {tag: result[0], time: result[1], message: result[2]}
+  end
+
+  sub_test_case "configure" do
+    test "check_default" do
+      create_driver "default"
+    end
+  end
+
+  sub_test_case "emit" do
+    test "with valid record" do
+      driver = emit "", "test", [
+        {a: 1, b: {c: 2, d: {e: 3, f:4}, g: [10, 20, 30]}, h: [], i: {}},
+        {a: {b: {c: 1, d: {e: 2, f:3}, g: [10, 20, 30]}, h: [], i: {}}}
+      ]
+
+      expect_message = {"a" => 1, "b.c" => 2, "b.d.e" => 3, "b.d.f" => 4, "b.g.0" => 10, "b.g.1" => 20, "b.g.2" => 30, "h" => nil, "i" => nil}
+      result = emitted driver, 0
+      assert_equal "test", result[:tag]
+      assert_equal expect_message, result[:message]
+
+      expect_message = {"a.b.c" => 1, "a.b.d.e" => 2, "a.b.d.f" => 3, "a.b.g.0" => 10, "a.b.g.1" => 20, "a.b.g.2" => 30, "a.h" => nil, "a.i" => nil}
+      result = emitted driver, 1
+      assert_equal "test", result[:tag]
+      assert_equal expect_message, result[:message]
+    end
+
+    test "with invalid record" do
+      driver = emit "", "test", ["message", {hoge: 1, fuga: {"test0" => 2, "test1" => 3}}]
+
+      expect_message = {}
+      result = emitted driver, 0
+      assert_equal expect_message, result[:message]
+
+      expect_message = {"hoge" => 1, "fuga.test0" => 2, "fuga.test1" => 3}
+      result = emitted driver, 1
+      assert_equal "test", result[:tag]
+      assert_equal expect_message, result[:message]
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require "test/unit"
+require "fluent/test"
+require "fluent/log"
+require "fluent/load"
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-nested-hash-filter
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- sugilog
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent Plugin for converting nested hash into flatten key-value pair.
+email:
+- sugilog@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Guardfile
+- README.md
+- Rakefile
+- fluent-plugin-nested-hash-filter.gemspec
+- lib/fluent/plugin/filter_nested_hash.rb
+- lib/fluent/plugin/nested_hash_filter/nested_object.rb
+- lib/fluent/plugin/out_nested_hash.rb
+- test/fluent/plugin/filter_nested_hash_test.rb
+- test/fluent/plugin/nested_hash_filter/nested_object_test.rb
+- test/fluent/plugin/out_nested_hash_test.rb
+- test/test_helper.rb
+homepage: https://github.com/sugilog/fluent-plugin-nested-hash-filter
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Fluent Plugin for converting nested hash into flatten key-value pair.
+test_files: []