fluent-plugin-flatten 0.0.10 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44fed39bc14936c3c382922850cf38ce9f88b0ab
-  data.tar.gz: 4d596b67719906871a23265501742fd4e5fdf1b1
+  metadata.gz: 5f89bacd8f75c114993634731bce29093a182fe7
+  data.tar.gz: 58defd52ece87e48bfeffa339591a10d79b256d8
 SHA512:
-  metadata.gz: 212995f7b1bc5c5e9d44b4f52d192151b862ce6cf0d5514bf682f7834774d1c540e1f07b960fa169355a41de8462da9c902e1deef777881d62eb010cf4162f8a
-  data.tar.gz: 42d74c0a6f831607c0e73636d1973f7975077c8f54776bc5b942bf8d9a308c91c4d8d2797d5982de37aba0069469e2519c5a9f5209e54381a68a943e7d2e1dac
+  metadata.gz: 4c13ccffa9ee50538ce8ac06448e2f5f1cb56b7d413d254964dd17ee6f42984e5038741230eee8e282a3ac90012d1973866df291442f01d084e41dc927e8a283
+  data.tar.gz: a10848bb4ec3b8e4ee37a90634a7b1e7e05255110c952210edd59ec0127e1ac21f351b58ef4c9aec153a8b70e4b56f3c7870407203caa6012c90bfb6501694ab

data/.gitignore

@@ -8,6 +8,7 @@ InstalledFiles
 _yardoc
 coverage
 doc/
+gemfiles/*.lock
 lib/bundler/man
 pkg
 rdoc

data/.travis.yml

@@ -0,0 +1,13 @@
+sudo: false
+language: ruby
+
+rvm:
+  - 2.1.10
+  - 2.2.5
+  - 2.3.1
+
+gemfile:
+  - gemfiles/fluentd_v0.14.gemfile
+
+script:
+  - bundle exec rake test

data/Appraisals

@@ -0,0 +1,3 @@
+appraise "fluentd v0.14" do
+  gem "fluentd", "~> 0.14.0"
+end

data/fluent-plugin-flatten.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-flatten'
-  gem.version       = '0.0.10'
+  gem.version       = '0.1.0'
   gem.authors       = ['Kentaro Kuribayashi']
   gem.email         = ['kentarok@gmail.com']
   gem.homepage      = 'http://github.com/kentaro/fluent-plugin-flatten'
@@ -13,8 +13,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
+  gem.add_development_dependency 'appraisal'
   gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'test-unit', '~> 3.1.0'
-  gem.add_runtime_dependency     'fluentd'
+  gem.add_development_dependency 'test-unit', '~> 3.0'
+  gem.add_runtime_dependency     'fluentd', ['>= 0.14.8', '< 2']
 end
-

data/gemfiles/fluentd_v0.14.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "fluentd", "~> 0.14.0"
+
+gemspec :path => "../"

data/lib/fluent/plugin/out_flatten.rb

@@ -1,29 +1,22 @@
 require 'json'
 
-module Fluent
-  class FlattenOutput < Output
+module Fluent::Plugin
+  class FlattenOutput < Fluent::Plugin::Output
     include Fluent::HandleTagNameMixin
     class Error < StandardError; end
 
     Fluent::Plugin.register_output('flatten', self)
 
-    # Define `router` method of v0.12 to support v0.10 or earlier
-    unless method_defined?(:router)
-      define_method("router") { Fluent::Engine }
-    end
+    helpers :event_emitter
 
-    config_param :key,        :string,
-                 :desc => <<-DESC
-The key is used to point a key whose value contains JSON-formatted string.
-DESC
-    config_param :inner_key,  :string, :default => 'value',
-                 :desc => <<-DESC
-This plugin sets `value` for this option as a default if it's not set.
-DESC
-    config_param :parse_json, :bool,   :default => true,
-                 :desc => "Parse json record."
-    config_param :replace_space_in_tag, :string,   :default => nil,
-                 :desc => "Replaces spaces in the resulting tag with the key passed"
+    desc "The key is used to point a key whose value contains JSON-formatted string."
+    config_param :key, :string
+    desc "This plugin sets `value` for this option as a default if it's not set."
+    config_param :inner_key, :string, default: 'value'
+    desc "Parse json record."
+    config_param :parse_json, :bool, default: true
+    desc "Replaces spaces in the resulting tag with the key passed"
+    config_param :replace_space_in_tag, :string, default: nil
 
     def configure(conf)
       super
@@ -34,11 +27,11 @@ DESC
           !add_tag_prefix    &&
           !add_tag_suffix
       )
-        raise ConfigError, "out_flatten: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set"
+        raise Fluent::ConfigError, "out_flatten: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set"
       end
     end
 
-    def emit(tag, es, chain)
+    def process(tag, es)
       es.each do |time, record|
         flattened = flatten(record)
 
@@ -52,8 +45,6 @@ DESC
           end
         end
       end
-
-      chain.next
     end
 
     def flatten(record)

data/test/plugin/test_out_flatten.rb

@@ -11,144 +11,161 @@ class FlattenOutputTest < Test::Unit::TestCase
     remove_tag_prefix test.
   ]
 
-  def create_driver(conf = DEFAULT_CONFIG, tag = 'test')
-    Fluent::Test::OutputTestDriver.new(Fluent::FlattenOutput, tag).configure(conf)
+  def create_driver(conf = DEFAULT_CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::FlattenOutput).configure(conf)
   end
 
-  def test_configure
-    # when `inner_key` option is not set
-    d1 = create_driver
-
-    assert_equal 'foo',                d1.instance.key
-    assert_equal 'flattened.',         d1.instance.add_tag_prefix
-    assert_equal /^test\./,            d1.instance.remove_tag_prefix
-    assert_equal 'value',              d1.instance.inner_key          # default value
-
-    # when `inner_key` is set
-    d2 = create_driver(%[
-      key               foo
-      add_tag_prefix    flattened.
-      remove_tag_prefix test.
-      inner_key         value_for_flat_key
-    ])
-
-    assert_equal 'foo',                d2.instance.key
-    assert_equal 'flattened.',         d2.instance.add_tag_prefix
-    assert_equal /^test\./,            d2.instance.remove_tag_prefix
-    assert_equal 'value_for_flat_key', d2.instance.inner_key
-
-    # when `parse_json` is false
-    d3 = create_driver(%[
-      key               foo
-      add_tag_prefix    flattened.
-      remove_tag_prefix test.
-      inner_key         value_for_flat_key
-      parse_json        false
-    ])
-
-    assert_equal 'foo',                d3.instance.key
-    assert_equal 'flattened.',         d3.instance.add_tag_prefix
-    assert_equal /^test\./,            d3.instance.remove_tag_prefix
-    assert_equal 'value_for_flat_key', d3.instance.inner_key
-    assert_equal false,         d3.instance.parse_json
-
-    # when mandatory keys not set
-    assert_raise(Fluent::ConfigError) do
-      create_driver(%[
-        key        foo
-        inner_key  value_for_keypath
+  sub_test_case "configure" do
+    test "when `inner_key` option is not set" do
+      d = create_driver
+
+      assert_equal 'foo',                d.instance.key
+      assert_equal 'flattened.',         d.instance.add_tag_prefix
+      assert_equal /^test\./,            d.instance.remove_tag_prefix
+      assert_equal 'value',              d.instance.inner_key          # default value
+    end
+
+    test "when `inner_key` is set" do
+      d = create_driver(%[
+        key               foo
+        add_tag_prefix    flattened.
+        remove_tag_prefix test.
+        inner_key         value_for_flat_key
       ])
+
+      assert_equal 'foo',                d.instance.key
+      assert_equal 'flattened.',         d.instance.add_tag_prefix
+      assert_equal /^test\./,            d.instance.remove_tag_prefix
+      assert_equal 'value_for_flat_key', d.instance.inner_key
+    end
+
+    test "when `parse_json` is false" do
+      d = create_driver(%[
+        key               foo
+        add_tag_prefix    flattened.
+        remove_tag_prefix test.
+        inner_key         value_for_flat_key
+        parse_json        false
+      ])
+
+      assert_equal 'foo',                d.instance.key
+      assert_equal 'flattened.',         d.instance.add_tag_prefix
+      assert_equal /^test\./,            d.instance.remove_tag_prefix
+      assert_equal 'value_for_flat_key', d.instance.inner_key
+      assert_equal false,         d.instance.parse_json
+    end
+
+    test "mandatory parameters are missing" do
+      assert_raise(Fluent::ConfigError) do
+        create_driver(%[
+          key        foo
+          inner_key  value_for_keypath
+        ])
+      end
     end
   end
 
-  def test_flatten
-    d = create_driver
+  sub_test_case "flatten" do
+    test "plain" do
+      d = create_driver
 
-    flattened = d.instance.flatten({ 'foo' => '{"bar" : "baz"}', 'hoge' => 'fuga' })
-    assert_equal({ 'foo.bar' => { 'value' => 'baz' } }, flattened)
+      flattened = d.instance.flatten({ 'foo' => '{"bar" : "baz"}', 'hoge' => 'fuga' })
+      assert_equal({ 'foo.bar' => { 'value' => 'baz' } }, flattened)
+    end
 
-    # XXX work-around
-    # fluentd seems to escape json value excessively
-    flattened = d.instance.flatten({ 'foo' => '{\"bar\" : \"baz\"}' })
-    assert_equal({ 'foo.bar' => { 'value' => 'baz' } }, flattened)
+    test "excessively escaped json value" do
+      d = create_driver
 
-    # when empty value is passed
-    flattened = d.instance.flatten({ 'foo' => '' })
-    assert_equal({}, flattened)
+      # XXX work-around
+      # fluentd seems to escape json value excessively
+      flattened = d.instance.flatten({ 'foo' => '{\"bar\" : \"baz\"}' })
+      assert_equal({ 'foo.bar' => { 'value' => 'baz' } }, flattened)
+    end
 
-    # when invalid json value is passed
-    flattened = d.instance.flatten({ 'foo' => '-' })
-    assert_equal({}, flattened)
+    test "empty" do
+      d = create_driver
+      flattened = d.instance.flatten({ 'foo' => '' })
+      assert_equal({}, flattened)
+    end
+
+    test "invalid json" do
+      d = create_driver
+      flattened = d.instance.flatten({ 'foo' => '-' })
+      assert_equal({}, flattened)
+    end
   end
 
-  def test_emit
-    # test1 default config
-    d1 = create_driver
+  sub_test_case "emit" do
+    test "default config" do
+      # test1 default config
+      d = create_driver
 
-    d1.run do
-      d1.emit( 'foo' => '{"bar" : "baz"}', 'hoge' => 'fuga' )
-      d1.emit( 'foo' => '{"bar" : {"qux" : "quux", "hoe" : "poe" }, "baz" : "bazz" }', 'hoge' => 'fuga' )
-    end
-    emits1 = d1.emits
-
-    assert_equal 4, emits1.count
-
-    # ["flattened.foo.bar", 1354689632, {"value"=>"baz"}]
-    assert_equal     'flattened.foo.bar', emits1[0][0]
-    assert_equal                   'baz', emits1[0][2]['value']
-
-    # ["flattened.foo.bar.qux", 1354689632, {"value"=>"quux"}]
-    assert_equal 'flattened.foo.bar.qux', emits1[1][0]
-    assert_equal                  'quux', emits1[1][2]['value']
-
-    # ["flattened.foo.bar.hoe", 1354689632, {"value"=>"poe"}]
-    assert_equal 'flattened.foo.bar.hoe', emits1[2][0]
-    assert_equal                   'poe', emits1[2][2]['value']
-
-    # ["flattened.foo.bar.baz", 1354689632, {"value"=>"bazz"}]
-    assert_equal     'flattened.foo.baz', emits1[3][0]
-    assert_equal                  'bazz', emits1[3][2]['value']
-
-    # test2 parse_json is set false 
-    d2 = create_driver(%[
-      key                  foo 
-      add_tag_prefix       flattened.
-      remove_tag_prefix    test.
-      parse_json           false
-      replace_space_in_tag _
-    ])
-
-    d2.run do
-      d2.emit( 'foo' => {'bar' => 'baz'}, 'hoge' => 'fuga' )
-      d2.emit( 'foo' => {'bar' => {'qux' => 'quux', 'hoe' => 'poe' }, 'baz' => 'bazz' }, 'hoge' => 'fuga' )
-      d2.emit( 'foo' => {'bar hoge' => 'baz', 'hoe baz' => 'poe'}, 'hoge' => 'fuga' )
+      d.run(default_tag: "test") do
+        d.feed( 'foo' => '{"bar" : "baz"}', 'hoge' => 'fuga' )
+        d.feed( 'foo' => '{"bar" : {"qux" : "quux", "hoe" : "poe" }, "baz" : "bazz" }', 'hoge' => 'fuga' )
+      end
+      events = d.events
+
+      assert_equal 4, events.count
+
+      # ["flattened.foo.bar", 1354689632, {"value"=>"baz"}]
+      assert_equal     'flattened.foo.bar', events[0][0]
+      assert_equal                   'baz', events[0][2]['value']
+
+      # ["flattened.foo.bar.qux", 1354689632, {"value"=>"quux"}]
+      assert_equal 'flattened.foo.bar.qux', events[1][0]
+      assert_equal                  'quux', events[1][2]['value']
+
+      # ["flattened.foo.bar.hoe", 1354689632, {"value"=>"poe"}]
+      assert_equal 'flattened.foo.bar.hoe', events[2][0]
+      assert_equal                   'poe', events[2][2]['value']
+
+      # ["flattened.foo.bar.baz", 1354689632, {"value"=>"bazz"}]
+      assert_equal     'flattened.foo.baz', events[3][0]
+      assert_equal                  'bazz', events[3][2]['value']
     end
-    emits2 = d2.emits
 
-    assert_equal 6, emits2.count
+    test "parse_json is set false" do
+      d = create_driver(%[
+        key                  foo
+        add_tag_prefix       flattened.
+        remove_tag_prefix    test.
+        parse_json           false
+        replace_space_in_tag _
+      ])
+
+      d.run(default_tag: "test") do
+        d.feed( 'foo' => {'bar' => 'baz'}, 'hoge' => 'fuga' )
+        d.feed( 'foo' => {'bar' => {'qux' => 'quux', 'hoe' => 'poe' }, 'baz' => 'bazz' }, 'hoge' => 'fuga' )
+        d.feed( 'foo' => {'bar hoge' => 'baz', 'hoe baz' => 'poe'}, 'hoge' => 'fuga' )
+      end
+      events = d.events
+
+      assert_equal 6, events.count
 
-    # ["flattened.foo.bar", 1354689632, {"value"=>"baz"}]
-    assert_equal          'flattened.foo.bar', emits2[0][0]
-    assert_equal                        'baz', emits2[0][2]['value']
+      # ["flattened.foo.bar", 1354689632, {"value"=>"baz"}]
+      assert_equal          'flattened.foo.bar', events[0][0]
+      assert_equal                        'baz', events[0][2]['value']
 
-    # ["flattened.foo.bar.qux_qux", 1354689632, {"value"=>"quux"}]
-    assert_equal      'flattened.foo.bar.qux', emits2[1][0]
-    assert_equal                       'quux', emits2[1][2]['value']
+      # ["flattened.foo.bar.qux_qux", 1354689632, {"value"=>"quux"}]
+      assert_equal      'flattened.foo.bar.qux', events[1][0]
+      assert_equal                       'quux', events[1][2]['value']
 
-    # ["flattened.foo.bar.hoe", 1354689632, {"value"=>"poe"}]
-    assert_equal      'flattened.foo.bar.hoe', emits2[2][0]
-    assert_equal                        'poe', emits2[2][2]['value']
+      # ["flattened.foo.bar.hoe", 1354689632, {"value"=>"poe"}]
+      assert_equal      'flattened.foo.bar.hoe', events[2][0]
+      assert_equal                        'poe', events[2][2]['value']
 
-    # ["flattened.foo.bar.baz", 1354689632, {"value"=>"bazz"}]
-    assert_equal          'flattened.foo.baz', emits2[3][0]
-    assert_equal                       'bazz', emits2[3][2]['value']
+      # ["flattened.foo.bar.baz", 1354689632, {"value"=>"bazz"}]
+      assert_equal          'flattened.foo.baz', events[3][0]
+      assert_equal                       'bazz', events[3][2]['value']
 
-    # ["flattened.foo.bar_hoge", 1354689632, {"value"=>"baz"}]
-    assert_equal     'flattened.foo.bar_hoge', emits2[4][0]
-    assert_equal                        'baz', emits2[4][2]['value']
+      # ["flattened.foo.bar_hoge", 1354689632, {"value"=>"baz"}]
+      assert_equal     'flattened.foo.bar_hoge', events[4][0]
+      assert_equal                        'baz', events[4][2]['value']
 
-    # ["flattened.foo.hoe_baz", 1354689632, {"value"=>"baz"}]
-    assert_equal      'flattened.foo.hoe_baz', emits2[5][0]
-    assert_equal                        'poe', emits2[5][2]['value']
+      # ["flattened.foo.hoe_baz", 1354689632, {"value"=>"baz"}]
+      assert_equal      'flattened.foo.hoe_baz', events[5][0]
+      assert_equal                        'poe', events[5][2]['value']
+    end
   end
 end

data/test/test_helper.rb

@@ -1,32 +1,9 @@
-require 'rubygems'
-require 'bundler'
-
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-
+require 'bundler/setup'
 require 'test/unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
 
 require 'fluent/test'
-
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
-
+require 'fluent/test/driver/output'
 require 'fluent/plugin/out_flatten'
-
-class Test::Unit::TestCase
-end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flatten
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.1.0
 platform: ruby
 authors:
 - Kentaro Kuribayashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-05 00:00:00.000000000 Z
+date: 2016-11-14 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -30,28 +44,34 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: Fluentd plugin to extract values for nested key paths and re-emit them
   as flat tag/record pairs.
 email:
@@ -61,11 +81,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
+- Appraisals
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - fluent-plugin-flatten.gemspec
+- gemfiles/fluentd_v0.14.gemfile
 - lib/fluent/plugin/out_flatten.rb
 - test/plugin/test_out_flatten.rb
 - test/test_helper.rb