fluent-plugin-record_splitter 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a19daed38b9a4be8b90350fd966c7b5c0eba4366
-  data.tar.gz: e1fffed00204b7cda4cd9784c0bf62e3e12f782f
+  metadata.gz: cd96d0a95cd0fea74f83b3696a5f613a6317871f
+  data.tar.gz: 776b7ed3e78ebfc77ac57afe9ae1aac0080841d1
 SHA512:
-  metadata.gz: 34402e3016d615bd6a390d086e0eb2d5ccdc470b28c2395d97215f4fd2aa83e039cb1e31b6837de00a2bf1a75bdc997a39a827b33f35843081bad42176bea42f
-  data.tar.gz: 6ecdc6bdd392aac400b89630c3afa019800e1ae8e4bb82415a6bb14e95696a5182bf2d256714178def64d907fcde96af3fed387fe78dfc39198fa6139a30046a
+  metadata.gz: 40689124abcdb86913e46d1eae027a2527b908f3f67ce63f65ed5b7092b63dfc9e5e46d0a579621a550badbaba000764592ed8e34a8854e71fddb66b9f8f093f
+  data.tar.gz: e015c41e2be6f89c13e80fa671064946a0ee30a14b965dd2915be7bde0101dfce2aac6ede69fe58ef2b5af2f075f191873314953fe3290d79be3985e9e6994d6

data/Gemfile

@@ -1,3 +1,3 @@
-source "http://rubygems.org"
+source 'http://rubygems.org'
 
 gemspec

data/README.md

@@ -3,6 +3,13 @@ fluent-plugin-record_splitter
 
 Output split array plugin for fluentd.
 
+## Dependence
+
+- td-agent v2(fluentd ~> 0.12.0)
+   - fluent-plugin-record_splitter < 0.1.6
+- td-agnet v3(fluentd ~> 0.14.0)
+   - fluent-plugin-record_splitter >= 0.1.6
+
 ## Installation
 
 ```
@@ -13,7 +20,7 @@ gem install fluent-plugin-record_splitter
 
     <match pattern>
       type record_splitter
-      tag foo.splitted
+      tag foo.split
       split_key target_field
       keep_keys ["common","general"]
     </match>
@@ -35,7 +42,7 @@ another configuration
 
     <match pattern>
       type record_splitter
-      tag foo.splitted
+      tag foo.split
       split_key target_field
       keep_other_key true
       remove_keys ["general"]

data/Rakefile

@@ -9,5 +9,4 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
-task :default => [:build]
-
+task default: [:build]

data/VERSION

@@ -1 +1 @@
-0.1.5
+0.2.0

data/fluent-plugin-record_splitter.gemspec

@@ -1,23 +1,25 @@
 # encoding: utf-8
-$:.push File.expand_path('../lib', __FILE__)
+
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.name        = "fluent-plugin-record_splitter"
-  gem.description = "output split array plugin for fluentd"
-  gem.homepage    = "https://github.com/ixixi/fluent-plugin-record_splitter"
+  gem.name        = 'fluent-plugin-record_splitter'
+  gem.description = 'output split array plugin for fluentd'
+  gem.homepage    = 'https://github.com/ixixi/fluent-plugin-record_splitter'
   gem.summary     = gem.description
-  gem.version     = File.read("VERSION").strip
-  gem.authors     = ["Yuri Odagiri"]
-  gem.email       = "ixixizko@gmail.com"
+  gem.version     = File.read('VERSION').strip
+  gem.authors     = ['Yuri Odagiri']
+  gem.email       = 'ixixizko@gmail.com'
   gem.has_rdoc    = false
   gem.license     = 'MIT'
   gem.files       = `git ls-files`.split("\n")
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
-  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
-  gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
-  gem.add_dependency "fluent-mixin-rewrite-tag-name"
-  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_dependency 'fluentd', ['>= 0.14.0', '< 2']
+  gem.add_dependency 'fluent-mixin-config-placeholders', '>= 0.3.0'
+  gem.add_dependency 'fluent-mixin-rewrite-tag-name'
+  gem.add_development_dependency 'rake', '>= 0.9.2'
+  gem.add_development_dependency 'test-unit'
 end

data/lib/fluent/plugin/out_record_splitter.rb

@@ -1,60 +1,70 @@
+require 'fluent/plugin/output'
 require 'fluent/mixin/config_placeholders'
 require 'fluent/mixin/rewrite_tag_name'
 
 module Fluent
-  class RecordSplitterOutput < Output
-    Fluent::Plugin.register_output('record_splitter', self)
-
-    config_param :tag, :string
-    config_param :remove_prefix, :string, :default => nil
-    config_param :add_prefix, :string, :default => nil
-    config_param :split_key, :string
-    config_param :keep_other_key, :bool, :default => false
-    config_param :keep_keys, :array, :default => []
-    config_param :remove_keys, :array, :default => []
-
-    include SetTagKeyMixin
-    include Fluent::Mixin::ConfigPlaceholders
-    include Fluent::HandleTagNameMixin
-    include Fluent::Mixin::RewriteTagName
-
-    def configure(conf)
-      super
-      if not @keep_keys.empty? and not @remove_keys.empty?
-        raise Fluent::ConfigError, 'Cannot set both keep_keys and remove_keys.'
-      end
-      if @keep_other_key and not @keep_keys.empty?
-        raise Fluent::ConfigError, 'Cannot set keep_keys when keep_other_key is true.'
-      end
-      if not @keep_other_key and not @remove_keys.empty?
-        raise Fluent::ConfigError, 'Cannot set remove_keys when keep_other_key is false.'
-      end
-      if not @tag and not @remove_prefix and not @add_prefix
-        raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
-      end
-      if @tag and (@remove_prefix or @add_prefix)
-        raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
+  module Plugin
+    class RecordSplitterOutput < Output
+      Fluent::Plugin.register_output('record_splitter', self)
+
+      config_param :tag, :string
+      config_param :remove_prefix, :string, default: nil
+      config_param :add_prefix, :string, default: nil
+      config_param :split_key, :string
+      config_param :keep_other_key, :bool, default: false
+      config_param :keep_keys, :array, default: []
+      config_param :remove_keys, :array, default: []
+
+      include SetTagKeyMixin
+      include Fluent::Mixin::ConfigPlaceholders
+      include Fluent::HandleTagNameMixin
+      include Fluent::Mixin::RewriteTagName
+
+      helpers :event_emitter
+
+      def multi_workers_ready?
+        true
       end
-    end
 
-    def emit(tag, es, chain)
-      emit_tag = tag.dup
-      es.each { |time, record|
-        filter_record(emit_tag, time, record)
-        if keep_other_key
-          common = record.reject{|key, value| key == @split_key or @remove_keys.include?(key) }
-        else
-          common = record.select{|key, value| @keep_keys.include?(key) }
+      def configure(conf)
+        super
+        if !@keep_keys.empty? && !@remove_keys.empty?
+          raise Fluent::ConfigError, 'Cannot set both keep_keys and remove_keys.'
+        end
+        if @keep_other_key && !@keep_keys.empty?
+          raise Fluent::ConfigError, 'Cannot set keep_keys when keep_other_key is true.'
+        end
+        if !@keep_other_key && !@remove_keys.empty?
+          raise Fluent::ConfigError, 'Cannot set remove_keys when keep_other_key is false.'
         end
-        if record.key?(@split_key)
-          record[@split_key].each{|v|
+        if !@tag && !@remove_prefix && !@add_prefix
+          raise Fluent::ConfigError, 'missing both of remove_prefix and add_prefix'
+        end
+        if @tag && (@remove_prefix || @add_prefix)
+          raise Fluent::ConfigError, 'both of tag and remove_prefix/add_prefix must not be specified'
+        end
+      end
+
+      def process(tag, es)
+        emit_tag = tag.dup
+
+        es.each do |time, record|
+          filter_record(emit_tag, time, record)
+
+          if @keep_other_key
+            common = record.reject { |key, _value| key == @split_key || @remove_keys.include?(key) }
+          else
+            common = record.select { |key, _value| @keep_keys.include?(key) }
+          end
+
+          next unless record.key?(@split_key)
+
+          record[@split_key].each do |v|
             v.merge!(common) unless common.empty?
-            router.emit(emit_tag, time, v.merge(common))
-          }
+            router.emit(emit_tag, time, v)
+          end
         end
-      }
-      chain.next
+      end
     end
-
   end
 end

data/test/test_out_record_splitter.rb

@@ -1,92 +1,106 @@
 require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'test/unit'
 require 'fluent/plugin/out_record_splitter'
 
-
 class RecordSplitterOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
-  CONFIG = %[
-    type record_splitter
-    tag foo.splitted
-  ]
+  CONFIG = %().freeze
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::RecordSplitterOutput, tag='test_tag').configure(conf)
+    Fluent::Test::Driver::Output
+      .new(Fluent::Plugin::RecordSplitterOutput)
+      .configure(conf)
+  end
+
+  def event_time
+    Time.parse('2017-06-01 00:11:22 UTC').to_i
   end
 
   def test_split_default
-    d = create_driver %[
-      type record_splitter
-      tag foo.splitted
+    d = create_driver %(
+      @type record_splitter
+      tag foo.split
       split_key target_field
-    ]
+    )
 
-    d.run do
-      d.emit('other'=>'foo','target_field' => [{'k1'=>'v1'},{'k2'=>'v2'}])
+    d.run(default_tag: 'test') do
+      d.feed(event_time,
+             'other' => 'foo',
+             'target_field' => [{ 'k1' => 'v1' }, { 'k2' => 'v2' }])
     end
 
     assert_equal [
-      {'k1'=>'v1'},
-      {'k2'=>'v2'},
-    ], d.records
+      ['foo.split', event_time, { 'k1' => 'v1' }],
+      ['foo.split', event_time, { 'k2' => 'v2' }]
+    ], d.events
   end
 
-
   def test_split_with_keep_other_key
-    d = create_driver %[
+    d = create_driver %(
       type record_splitter
-      tag foo.splitted
+      tag foo.split
       split_key target_field
       keep_other_key true
-    ]
+    )
 
-    d.run do
-      d.emit('common'=>'c','general'=>'g','other'=>'foo','target_field' => [{'k1'=>'v1'},{'k2'=>'v2'}])
+    d.run(default_tag: 'test') do
+      d.feed(event_time,
+             'common' => 'c', 'general' => 'g', 'other' => 'foo',
+             'target_field' => [{ 'k1' => 'v1' }, { 'k2' => 'v2' }])
     end
 
     assert_equal [
-      {'common'=>'c','general'=>'g','other'=>'foo','k1'=>'v1'},
-      {'common'=>'c','general'=>'g','other'=>'foo','k2'=>'v2'},
-    ], d.records
+      ['foo.split', event_time,
+       { 'common' => 'c', 'general' => 'g', 'other' => 'foo', 'k1' => 'v1' }],
+      ['foo.split', event_time,
+       { 'common' => 'c', 'general' => 'g', 'other' => 'foo', 'k2' => 'v2' }]
+    ], d.events
   end
 
   def test_split_with_keep_other_key_and_remove_key
-    d = create_driver %[
+    d = create_driver %(
       type record_splitter
-      tag foo.splitted
+      tag foo.split
       split_key target_field
       keep_other_key true
       remove_keys ["general","other"]
-    ]
+    )
 
-    d.run do
-      d.emit('common'=>'c','general'=>'g','other'=>'foo','target_field' => [{'k1'=>'v1'},{'k2'=>'v2'}])
+    d.run(default_tag: 'test') do
+      d.feed(event_time,
+             'common' => 'c', 'general' => 'g', 'other' => 'foo',
+             'target_field' => [{ 'k1' => 'v1' }, { 'k2' => 'v2' }])
     end
 
     assert_equal [
-      {'common'=>'c','k1'=>'v1'},
-      {'common'=>'c','k2'=>'v2'},
-    ], d.records
+      ['foo.split', event_time, { 'common' => 'c', 'k1' => 'v1' }],
+      ['foo.split', event_time, { 'common' => 'c', 'k2' => 'v2' }]
+    ], d.events
   end
 
   def test_split_with_keep_keys
-    d = create_driver %[
+    d = create_driver %(
       type record_splitter
-      tag foo.splitted
+      tag foo.split
       split_key target_field
       keep_keys ["common","general"]
-    ]
+    )
 
-    d.run do
-      d.emit('common'=>'c','general'=>'g','other'=>'foo','target_field' => [{'k1'=>'v1'},{'k2'=>'v2'}])
+    d.run(default_tag: 'test') do
+      d.feed(event_time,
+             'common' => 'c', 'general' => 'g', 'other' => 'foo',
+             'target_field' => [{ 'k1' => 'v1' }, { 'k2' => 'v2' }])
     end
 
     assert_equal [
-      {'common'=>'c','general'=>'g','k1'=>'v1'},
-      {'common'=>'c','general'=>'g','k2'=>'v2'},
-    ], d.records
+      ['foo.split', event_time,
+       { 'common' => 'c', 'general' => 'g', 'k1' => 'v1' }],
+      ['foo.split', event_time,
+       { 'common' => 'c', 'general' => 'g', 'k2' => 'v2' }]
+    ], d.events
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record_splitter
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Yuri Odagiri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-09 00:00:00.000000000 Z
+date: 2017-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -72,6 +72,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: output split array plugin for fluentd
 email: ixixizko@gmail.com
 executables: []
@@ -106,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: output split array plugin for fluentd