fluent-plugin-copy_ex 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d312162cb51f1344bd45c89909d6188ff8d2acd
-  data.tar.gz: 7790f7b58b7a0367a25af1afcea4c9e0ea836989
+  metadata.gz: 9594127ab0102725fc88886986e3934234088aa1
+  data.tar.gz: 78c257e8b902c9c82cc3a973ca89fcaea447e2d7
 SHA512:
-  metadata.gz: 06f889b19eb0c0f38af2e907ce39c5f7f1af4172a831ff0cae885e833286f8212401db19f6cd0d9f570aa81453508efac2bd08309ed5d177e27928cea76a4699
-  data.tar.gz: 8d7eaea66f06c7abdab6c750a2eb6e23b12564c33228672d59d21767f97733a839e931bfb536722725ef228b1a7c36eba23624c6dbd514be8ef8a06138abfebe
+  metadata.gz: c12758f99c7876d8d85b0309f29a4cc07281033f9df0040baba572a084cec05dd8852e6a49947cdf372f7b161d3c39802c6375a5d3024a1845bd6aa186df801d
+  data.tar.gz: ac50e1b57fb77b181a57bba289835a32ba9edc8cd1918403d6bd68c02429ccae5e85f3c62542555f27be171c5c378eb8152ce44bac316dc7c7e5b9761b62d8fa

data/.travis.yml

@@ -1,8 +1,9 @@
 rvm:
-  - 2.1.*
   - 2.2.*
-  - 2.3.0
+  - 2.3.*
+  - 2.4.*
 gemfile:
   - Gemfile
+  - Gemfile.v0.12
 before_install:
   - gem update bundler

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.0 (2017-11-25)
+
+* Fix v0.14 support was not working well
+
 ## 0.0.3 (2017-02-06)
 
 * Support v0.14

data/Gemfile.v0.12

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gemspec
+gem 'fluentd', '~> 0.12.0'

data/Rakefile

@@ -4,7 +4,7 @@ require "bundler/gem_tasks"
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/*.rb'
+  test.pattern = 'test/**/test_*.rb'
   test.verbose = true
   test.warning = false
 end

data/example/stdout_and_file.conf

@@ -0,0 +1,25 @@
+<source>
+  @type dummy
+  tag "test.*"
+</source>
+<match **>
+  @type copy_ex
+  <store>
+    @type stdout
+  </store>
+  <store>
+    @type file
+    append true
+    path "output.log"
+    time_slice_format test-%Y-%m-%d
+    <format>
+      @type json
+      localtime false
+    </format>
+    <buffer>
+      @type memory
+      flush_mode immediate
+      flush_at_shutdown true
+    </buffer>
+  </store>
+</match>

data/example/stdout_and_stdout.conf

@@ -0,0 +1,14 @@
+<source>
+  @type dummy
+  tag test
+</source>
+
+<match **>
+  @type copy_ex
+  <store ignore_error>
+     @type stdout
+  </store>
+  <store ignore_error>
+     @type stdout
+  </store>
+</match>

data/fluent-plugin-copy_ex.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-copy_ex"
-  s.version     = "0.0.3"
+  s.version     = "0.1.0"
   s.authors     = ["Naotoshi Seo"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-copy_ex"

data/lib/fluent/plugin/out_copy_ex.rb

@@ -1,81 +1,7 @@
-require 'fluent/output'
-
-module Fluent
-  class CopyExOutput < MultiOutput
-    Plugin.register_output('copy_ex', self)
-
-    config_param :deep_copy, :bool, :default => false
-
-    def initialize
-      super
-      @outputs = []
-      @ignore_errors = []
-      @emit_procs = []
-    end
-
-    attr_reader :outputs, :ignore_errors
-
-    def configure(conf)
-      super
-      conf.elements.select {|e|
-        e.name == 'store'
-      }.each {|e|
-        type = e['@type'] || e['type']
-        unless type
-          raise ConfigError, "Missing '@type' parameter on <store> directive"
-        end
-        log.debug "adding store type=#{type.dump}"
-
-        output = Plugin.new_output(type)
-        output.configure(e)
-        emit_proc = if output.respond_to?(:emit_events)
-                      Proc.new {|output, tag, es, _chain| output.emit_events(tag, es)}
-                    else
-                      Proc.new {|output, tag, es, _chain| output.emit(tag, es, NullOutputChain.instance)}
-                    end
-        @emit_procs << emit_proc
-        @outputs << output
-
-        @ignore_errors << (e.arg == "ignore_error")
-      }
-    end
-
-    def start
-      @outputs.each {|o|
-        o.start
-      }
-    end
-
-    def shutdown
-      @outputs.each {|o|
-        o.shutdown
-      }
-    end
-
-    def emit(tag, es, chain)
-      unless es.repeatable?
-        m = MultiEventStream.new
-        es.each {|time,record|
-          m.add(time, record)
-        }
-        es = m
-      end
-
-      # Here, we do not use OutputChain for custom
-      @outputs.each_index do |idx|
-        _es = @deep_copy ? es.dup : es
-        begin
-          @emit_procs[idx].call(@outputs[idx], tag, _es, NullOutputChain.instance)
-        rescue => e
-          if @ignore_errors[idx]
-            log.error :error_class => e.class, :error => e.message
-          else
-            raise e
-          end
-        end
-      end
-
-      chain.next
-    end
-  end
+require 'fluent/version'
+major, minor, patch = Fluent::VERSION.split('.').map(&:to_i)
+if major > 0 || (major == 0 && minor >= 14)
+  require_relative 'out_copy_ex/v14'
+else
+  require_relative 'out_copy_ex/v12'
 end

data/lib/fluent/plugin/out_copy_ex/v12.rb

@@ -0,0 +1,49 @@
+require 'fluent/plugin/out_copy'
+
+module Fluent
+  class CopyExOutput < CopyOutput
+    Plugin.register_output('copy_ex', self)
+
+    attr_reader :ignore_errors
+
+    def initialize
+      super
+      @ignore_errors = []
+    end
+
+    def configure(conf)
+      super
+
+      conf.elements.select {|e|
+        e.name == 'store'
+      }.each {|e|
+        @ignore_errors << (e.arg == "ignore_error")
+      }
+    end
+
+    def emit(tag, es, chain)
+      unless es.repeatable?
+        m = MultiEventStream.new
+        es.each {|time,record|
+          m.add(time, record)
+        }
+        es = m
+      end
+
+      # Here, we do not use OutputChain for custom
+      outputs.each.with_index do |output, idx|
+        begin
+          output.emit(tag, @deep_copy ? es.dup : es, NullOutputChain.instance)
+        rescue => e
+          if @ignore_errors[idx]
+            log.error :error_class => e.class, :error => e.message
+          else
+            raise e
+          end
+        end
+      end
+
+      chain.next
+    end
+  end
+end

data/lib/fluent/plugin/out_copy_ex/v14.rb

@@ -0,0 +1,46 @@
+require 'fluent/plugin/out_copy'
+
+module Fluent::Plugin
+  class CopyExOutput < CopyOutput
+    Fluent::Plugin.register_output('copy_ex', self)
+
+    attr_reader :ignore_errors
+
+    def initialize
+      super
+      @ignore_errors = []
+    end
+
+    def configure(conf)
+      super
+
+      conf.elements.select {|e|
+        e.name == 'store'
+      }.each {|e|
+        @ignore_errors << (e.arg == "ignore_error")
+      }
+    end
+
+    def process(tag, es)
+      unless es.repeatable?
+        m = Fluent::MultiEventStream.new
+        es.each {|time,record|
+          m.add(time, record)
+        }
+        es = m
+      end
+
+      outputs.each.with_index do |output, idx|
+        begin
+          output.emit_events(tag, @deep_copy ? es.dup : es)
+        rescue => e
+          if @ignore_errors[idx]
+            log.error :error_class => e.class, :error => e.message
+          else
+            raise e
+          end
+        end
+      end
+    end
+  end
+end

data/test/{plugin/test_out_copy_ex.rb → out_copy_ex/v12.rb}

@@ -1,10 +1,9 @@
-require 'fluent/test'
+require_relative '../helper'
 
 class CopyExOutputTest < Test::Unit::TestCase
   class << self
     def startup
-      spec = Gem::Specification.find { |s| s.name == 'fluentd' }
-      $LOAD_PATH.unshift File.join(spec.full_gem_path, 'test', 'scripts')
+      $LOAD_PATH.unshift "#{Gem.loaded_specs['fluentd'].full_gem_path}/test/scripts"
       require 'fluent/plugin/out_test'
     end
 
@@ -17,10 +16,6 @@ class CopyExOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  def config_element(name = 'test', argument = '', params = {}, elements = [])
-    Fluent::Config::Element.new(name, argument, params, elements)
-  end
-
   CONFIG = %[
     <store>
       type test
@@ -60,9 +55,9 @@ class CopyExOutputTest < Test::Unit::TestCase
 
     outputs = d.instance.outputs
     assert_equal 3, outputs.size
-    assert_equal Fluent::Plugin::TestOutput, outputs[0].class
-    assert_equal Fluent::Plugin::TestOutput, outputs[1].class
-    assert_equal Fluent::Plugin::TestOutput, outputs[2].class
+    assert_equal Fluent::TestOutput, outputs[0].class
+    assert_equal Fluent::TestOutput, outputs[1].class
+    assert_equal Fluent::TestOutput, outputs[2].class
     assert_equal "c0", outputs[0].name
     assert_equal "c1", outputs[1].name
     assert_equal "c2", outputs[2].name
@@ -88,39 +83,35 @@ class CopyExOutputTest < Test::Unit::TestCase
 
     d.instance.outputs.each {|o|
       assert_equal [
-          [time, {"a"=>1}],
-          [time, {"a"=>2}],
-        ], o.events
+        [time, {"a"=>1}],
+        [time, {"a"=>2}],
+      ], o.events
+    }
+
+    d.instance.outputs.each {|o|
+      assert_not_nil o.router
     }
   end
 
   def test_msgpack_es_emit_bug
     d = Fluent::Test::OutputTestDriver.new(Fluent::CopyExOutput)
 
-    emit_procs = []
     outputs = %w(p1 p2).map do |pname|
       p = Fluent::Plugin.new_output('test')
-      p.configure(config_element('ROOT', '', {'name' => pname}))
+      p.configure('name' => pname)
       p.define_singleton_method(:emit) do |tag, es, chain|
         es.each do |time, record|
           super(tag, [[time, record]], chain)
         end
       end
-      emit_proc = if p.respond_to?(:emit_events)
-                    Proc.new {|p, tag, es, _chain| p.emit_events(tag, es)}
-                  else
-                    Proc.new {|p, tag, es, _chain| p.emit(tag, es, NullOutputChain.instance)}
-                  end
-      emit_procs << emit_proc
       p
     end
 
     d.instance.instance_eval { @outputs = outputs }
-    d.instance.instance_eval { @emit_procs = emit_procs }
 
     es = if defined?(MessagePack::Packer)
            time = Time.parse("2013-05-26 06:37:22 UTC").to_i
-           packer = MessagePack::Packer.new
+           packer = Fluent::Engine.msgpack_factory.packer
            packer.pack([time, {"a" => 1}])
            packer.pack([time, {"a" => 2}])
            Fluent::MessagePackEventStream.new(packer.to_s)
@@ -142,42 +133,30 @@ class CopyExOutputTest < Test::Unit::TestCase
   def create_event_test_driver(is_deep_copy = false)
     deep_copy_config = %[
 deep_copy true
-]
+    ]
 
     output1 = Fluent::Plugin.new_output('test')
-    output1.configure(config_element('ROOT', '', {'name' => 'output1'}))
-    output1.define_singleton_method(:emit_events) do |tag, es|
+    output1.configure('name' => 'output1')
+    output1.define_singleton_method(:emit) do |tag, es, chain|
       es.each do |time, record|
         record['foo'] = 'bar'
-        super(tag, [[time, record]])
+        super(tag, [[time, record]], chain)
       end
     end
-    proc1 = if output1.respond_to?(:emit_events)
-                  Proc.new {|output1, tag, es, _chain| output1.emit_events(tag, es)}
-                else
-                  Proc.new {|output1, tag, es, _chain| output1.emit(tag, es, NullOutputChain.instance)}
-                end
 
     output2 = Fluent::Plugin.new_output('test')
-    output2.configure(config_element('ROOT', '', {'name' => 'output2'}))
-    output2.define_singleton_method(:emit_events) do |tag, es|
+    output2.configure('name' => 'output2')
+    output2.define_singleton_method(:emit) do |tag, es, chain|
       es.each do |time, record|
-        super(tag, [[time, record]])
+        super(tag, [[time, record]], chain)
       end
     end
-    proc2 = if output2.respond_to?(:emit_events)
-                  Proc.new {|output2, tag, es, _chain| output2.emit_events(tag, es)}
-                else
-                  Proc.new {|output2, tag, es, _chain| output2.emit(tag, es, NullOutputChain.instance)}
-                end
 
     outputs = [output1, output2]
-    emit_procs = [proc1, proc2]
 
     d = Fluent::Test::OutputTestDriver.new(Fluent::CopyExOutput)
     d = d.configure(deep_copy_config) if is_deep_copy
     d.instance.instance_eval { @outputs = outputs }
-    d.instance.instance_eval { @emit_procs = emit_procs }
     d
   end
 
@@ -241,4 +220,3 @@ deep_copy true
     assert_nothing_raised { d.emit({"a"=>1}, time) }
   end
 end
-

data/test/out_copy_ex/v14.rb

@@ -0,0 +1,203 @@
+require_relative '../helper'
+require "fluent/test/driver/multi_output"
+
+class CopyExOutputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      $LOAD_PATH.unshift "#{Gem.loaded_specs['fluentd'].full_gem_path}/test/scripts"
+      require 'fluent/plugin/out_test'
+      require 'fluent/plugin/out_test2'
+    end
+
+    def shutdown
+      $LOAD_PATH.shift
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def config_element(name = 'test', argument = '', params = {}, elements = [])
+    Fluent::Config::Element.new(name, argument, params, elements)
+  end
+
+  CONFIG = %[
+    <store>
+      @type test
+      name c0
+    </store>
+    <store>
+      @type test2
+      name c1
+    </store>
+    <store>
+      @type test
+      name c2
+    </store>
+  ]
+
+  IGNORE_ERROR_CONFIG = %[
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_error>
+      @type test
+      name c1
+    </store>
+    <store>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyExOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    outputs = d.instance.outputs
+    assert_equal 3, outputs.size
+    assert_equal Fluent::Plugin::TestOutput, outputs[0].class
+    assert_equal Fluent::Plugin::Test2Output, outputs[1].class
+    assert_equal Fluent::Plugin::TestOutput, outputs[2].class
+    assert_equal "c0", outputs[0].name
+    assert_equal "c1", outputs[1].name
+    assert_equal "c2", outputs[2].name
+  end
+
+  def test_configure_ignore_error
+    d = create_driver(IGNORE_ERROR_CONFIG)
+
+    outputs = d.instance.outputs
+    ignore_errors = d.instance.ignore_errors
+    assert_equal outputs.size, ignore_errors.size
+    assert_equal true,  ignore_errors[0]
+    assert_equal true,  ignore_errors[1]
+    assert_equal false, ignore_errors[2]
+  end
+
+  def test_feed_events
+    d = create_driver
+
+    assert !d.instance.outputs[0].has_router?
+    assert_not_nil d.instance.outputs[1].router
+    assert !d.instance.outputs[2].has_router?
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {"a" => 1})
+      d.feed(time, {"a" => 2})
+    end
+
+    d.instance.outputs.each {|o|
+      assert_equal [ [time, {"a"=>1}], [time, {"a"=>2}] ], o.events
+    }
+  end
+
+  def test_msgpack_unpacker_cache_bug_for_msgpack_event_stream
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    source = Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"a" => 2}] ])
+    es = Fluent::MessagePackEventStream.new(source.to_msgpack_stream)
+
+    d.run(default_tag: 'test') do
+      d.feed(es)
+    end
+
+    d.instance.outputs.each { |o|
+      assert_equal [ [time, {"a"=>1}], [time, {"a"=>2}] ], o.events
+    }
+  end
+
+  def create_event_test_driver(does_deep_copy = false)
+    config = %[
+      deep_copy #{does_deep_copy}
+      <store>
+        @type test
+        name output1
+      </store>
+      <store>
+        @type test
+        name output2
+      </store>
+    ]
+
+    d = Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyExOutput).configure(config)
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      es.each do |time, record|
+        record['foo'] = 'bar'
+      end
+      super(tag, es)
+    end
+    d
+  end
+
+  time = Fluent::EventTime.parse("2013-05-26 06:37:22 UTC")
+  mes0 = Fluent::MultiEventStream.new
+  mes0.add(time, {"a" => 1})
+  mes0.add(time, {"b" => 1})
+  mes1 = Fluent::MultiEventStream.new
+  mes1.add(time, {"a" => 1})
+  mes1.add(time, {"b" => 1})
+
+  data(
+    "OneEventStream without deep_copy"   => [false, Fluent::OneEventStream.new(time, {"a" => 1})],
+    "OneEventStream with deep_copy"      => [true,  Fluent::OneEventStream.new(time, {"a" => 1})],
+    "ArrayEventStream without deep_copy" => [false, Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
+    "ArrayEventStream with deep_copy"    => [true,  Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
+    "MultiEventStream without deep_copy" => [false, mes0],
+    "MultiEventStream with deep_copy"    => [true,  mes1],
+  )
+  def test_deep_copy_controls_shallow_or_deep_copied(data)
+    does_deep_copy, es = data
+
+    d = create_event_test_driver(does_deep_copy)
+
+    d.run(default_tag: 'test') do
+      d.feed(es)
+    end
+
+    events = d.instance.outputs.map(&:events)
+
+    if does_deep_copy
+      events[0].each_with_index do |entry0, i|
+        record0 = entry0.last
+        record1 = events[1][i].last
+
+        assert{ record0.object_id != record1.object_id }
+        assert_equal "bar", record0["foo"]
+        assert !record1.has_key?("foo")
+      end
+    else
+      events[0].each_with_index do |entry0, i|
+        record0 = entry0.last
+        record1 = events[1][i].last
+
+        assert{ record0.object_id == record1.object_id }
+        assert_equal "bar", record0["foo"]
+        assert_equal "bar", record1["foo"]
+      end
+    end
+  end
+
+  def test_ignore_error
+    d = create_driver(IGNORE_ERROR_CONFIG)
+
+    # override to raise an error
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      raise ArgumentError, 'Failed'
+    end
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_nothing_raised do
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>1})
+      end
+    end
+  end
+end

data/test/test_out_copy_ex.rb

@@ -0,0 +1,9 @@
+require_relative "helper"
+
+require 'fluent/version'
+major, minor, patch = Fluent::VERSION.split('.').map(&:to_i)
+if major > 0 || (major == 0 && minor >= 14)
+  require_relative 'out_copy_ex/v14'
+else
+  require_relative 'out_copy_ex/v12'
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-copy_ex
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-06 00:00:00.000000000 Z
+date: 2017-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -91,14 +91,20 @@ files:
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
+- Gemfile.v0.12
 - LICENSE
 - README.md
 - Rakefile
-- example/fluent.conf
+- example/stdout_and_file.conf
+- example/stdout_and_stdout.conf
 - fluent-plugin-copy_ex.gemspec
 - lib/fluent/plugin/out_copy_ex.rb
+- lib/fluent/plugin/out_copy_ex/v12.rb
+- lib/fluent/plugin/out_copy_ex/v14.rb
 - test/helper.rb
-- test/plugin/test_out_copy_ex.rb
+- test/out_copy_ex/v12.rb
+- test/out_copy_ex/v14.rb
+- test/test_out_copy_ex.rb
 homepage: https://github.com/sonots/fluent-plugin-copy_ex
 licenses:
 - MIT
@@ -119,10 +125,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: fluent-plugin-copy_ex
-rubygems_version: 2.5.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Fluentd out_copy extension
 test_files:
 - test/helper.rb
-- test/plugin/test_out_copy_ex.rb
+- test/out_copy_ex/v12.rb
+- test/out_copy_ex/v14.rb
+- test/test_out_copy_ex.rb

data/example/fluent.conf

@@ -1,9 +0,0 @@
-<match **>
-  type copy_ex
-  <store ignore_error>
-     type stdout
-  </store>
-  <store ignore_error>
-     type stdout
-  </store>
-</match>