fluentd 1.17.0 → 1.17.1

data/test/test_output.rb

@@ -1,278 +0,0 @@
-require_relative 'helper'
-require 'fluent/test'
-require 'fluent/output'
-require 'fluent/output_chain'
-require 'fluent/plugin/buffer'
-require 'timecop'
-require 'flexmock/test_unit'
-
-module FluentOutputTest
-  include Fluent
-  include FlexMock::TestCase
-
-  class BufferedOutputTest < ::Test::Unit::TestCase
-    include FluentOutputTest
-
-    class << self
-      def startup
-        $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), 'scripts'))
-        require 'fluent/plugin/out_test'
-        require 'fluent/plugin/out_test2'
-      end
-
-      def shutdown
-        $LOAD_PATH.shift
-      end
-    end
-
-    def setup
-      Fluent::Test.setup
-    end
-
-    CONFIG = %[]
-
-    def create_driver(conf=CONFIG)
-      Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
-        def write(chunk)
-          chunk.read
-        end
-      end.configure(conf)
-    end
-
-    def test_configure
-      # default
-      d = create_driver
-      assert_equal 'memory', d.instance.buffer_type
-      assert_equal 60, d.instance.flush_interval
-      assert_equal false, d.instance.disable_retry_limit
-      assert_equal 17, d.instance.retry_limit
-      assert_equal 1.0, d.instance.retry_wait
-      assert_equal nil, d.instance.max_retry_wait
-      assert_equal 1.0, d.instance.retry_wait
-      assert_equal 1, d.instance.num_threads
-      assert_equal 1, d.instance.queued_chunk_flush_interval
-
-      # max_retry_wait
-      d = create_driver(CONFIG + %[max_retry_wait 4])
-      assert_equal 4, d.instance.max_retry_wait
-
-      # disable_retry_limit
-      d = create_driver(CONFIG + %[disable_retry_limit true])
-      assert_equal true, d.instance.disable_retry_limit
-
-      #### retry_state cares it
-      # # retry_wait is converted to Float for calc_retry_wait
-      # d = create_driver(CONFIG + %[retry_wait 1s])
-      # assert_equal Float, d.instance.retry_wait.class
-    end
-
-    class FormatterInjectTestOutput < Fluent::Output
-      def initialize
-        super
-        @formatter = nil
-      end
-    end
-    def test_start
-      i = FormatterInjectTestOutput.new
-      i.configure(config_element('ROOT', '', {}, [config_element('inject', '', {'hostname_key' => "host"})]))
-      assert_nothing_raised do
-        i.start
-      end
-    end
-
-    def create_mock_driver(conf=CONFIG)
-      Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
-        attr_accessor :submit_flush_threads
-
-        def start_mock
-          @started = false
-          start
-          # ensure OutputThread to start successfully
-          submit_flush
-          sleep 0.5
-          while !@started
-            submit_flush
-            sleep 0.5
-          end
-        end
-
-        def try_flush
-          @started = true
-          @submit_flush_threads ||= {}
-          @submit_flush_threads[Thread.current] ||= 0
-          @submit_flush_threads[Thread.current] += 1
-        end
-
-        def write(chunk)
-          chunk.read
-        end
-      end.configure(conf)
-    end
-
-    def test_secondary
-      d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
-        def write(chunk)
-          chunk.read
-        end
-      end
-
-      mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output")
-      d.configure(CONFIG + %[
-        <secondary>
-          type test2
-          name c0
-        </secondary>
-      ])
-
-      assert_not_nil d.instance.instance_variable_get(:@secondary).router
-    end
-
-    def test_secondary_with_no_warn_log
-      # ObjectBufferedOutput doesn't implement `custom_filter`
-      d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
-
-      mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output").never
-      d.configure(CONFIG + %[
-        <secondary>
-          type test2
-          name c0
-        </secondary>
-      ])
-
-      assert_not_nil d.instance.instance_variable_get(:@secondary).router
-    end
-
-    test 'BufferQueueLimitError compatibility' do
-      assert_equal Fluent::Plugin::Buffer::BufferOverflowError, Fluent::BufferQueueLimitError
-    end
-  end
-
-  class ObjectBufferedOutputTest < ::Test::Unit::TestCase
-    include FluentOutputTest
-
-    def setup
-      Fluent::Test.setup
-    end
-
-    CONFIG = %[]
-
-    def create_driver(conf=CONFIG)
-      Fluent::Test::OutputTestDriver.new(Fluent::ObjectBufferedOutput).configure(conf, true)
-    end
-
-    def test_configure
-      # default
-      d = create_driver
-      assert_equal true, d.instance.time_as_integer
-    end
-  end
-
-  class TimeSlicedOutputTest < ::Test::Unit::TestCase
-    include FluentOutputTest
-    include FlexMock::TestCase
-
-    def setup
-      Fluent::Test.setup
-      FileUtils.rm_rf(TMP_DIR)
-      FileUtils.mkdir_p(TMP_DIR)
-    end
-
-    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/time_sliced_output")
-
-    CONFIG = %[
-      buffer_path #{TMP_DIR}/foo
-      time_slice_format %Y%m%d%H
-    ]
-
-    class TimeSlicedOutputTestPlugin < Fluent::TimeSlicedOutput
-      attr_reader :written_chunk_keys, :errors_in_write
-      def initialize
-        super
-        @written_chunk_keys = []
-        @errors_in_write = []
-      end
-
-      def configure(conf)
-        super
-
-        @formatter = Fluent::Plugin.new_formatter('out_file')
-        @formatter.configure(conf)
-      end
-
-      def format(tag, time, record)
-        @formatter.format(tag, time, record)
-      end
-      def write(chunk)
-        @written_chunk_keys << chunk.key
-        true
-      rescue => e
-        @errors_in_write << e
-      end
-    end
-
-    def create_driver(conf=CONFIG)
-      Fluent::Test::TimeSlicedOutputTestDriver.new(TimeSlicedOutputTestPlugin).configure(conf, true)
-    end
-
-    data(:none => '',
-         :utc => "utc",
-         :localtime => 'localtime',
-         :timezone => 'timezone +0000')
-    test 'configure with timezone related parameters' do |param|
-      assert_nothing_raised {
-        create_driver(CONFIG + param)
-      }
-    end
-
-    sub_test_case "test emit" do
-      setup do
-        @time = Time.parse("2011-01-02 13:14:15 UTC")
-        Timecop.freeze(@time)
-        @newline = if Fluent.windows?
-                     "\r\n"
-                   else
-                     "\n"
-                   end
-      end
-
-      teardown do
-        Timecop.return
-      end
-
-      test "emit with invalid event" do
-        d = create_driver
-        d.instance.start
-        d.instance.after_start
-        assert_raise ArgumentError, "time must be a Fluent::EventTime (or Integer)" do
-          d.instance.emit_events('test', OneEventStream.new('string', 10))
-        end
-      end
-
-      test "plugin can get key of chunk in #write" do
-        d = create_driver
-        d.instance.start
-        d.instance.after_start
-        d.instance.emit_events('test', OneEventStream.new(event_time("2016-11-08 17:44:30 +0900"), {"message" => "yay"}))
-        d.instance.force_flush
-        waiting(10) do
-          sleep 0.1 until d.instance.written_chunk_keys.size == 1
-        end
-        assert_equal [], d.instance.errors_in_write
-        assert_equal ["2016110808"], d.instance.written_chunk_keys # default timezone is UTC
-      end
-
-      test "check formatted time compatibility with utc. Should Z, not +00:00" do
-        d = create_driver(CONFIG + %[
-          utc
-          include_time_key
-        ])
-        time = Time.parse("2016-11-08 12:00:00 UTC").to_i
-        d.emit({"a" => 1}, time)
-        d.expect_format %[2016-11-08T12:00:00Z\ttest\t{"a":1,"time":"2016-11-08T12:00:00Z"}#{@newline}]
-        d.run
-      end
-    end
-  end
-end

data/test/test_plugin.rb

@@ -1,251 +0,0 @@
-require_relative 'helper'
-
-require 'fluent/plugin'
-require 'fluent/plugin/input'
-require 'fluent/plugin/filter'
-require 'fluent/plugin/output'
-require 'fluent/plugin/buffer'
-require 'fluent/plugin/parser'
-require 'fluent/plugin/formatter'
-require 'fluent/plugin/storage'
-
-class PluginTest < Test::Unit::TestCase
-  class Dummy1Input < Fluent::Plugin::Input
-    Fluent::Plugin.register_input('plugin_test_dummy1', self)
-  end
-  class Dummy2Input < Fluent::Plugin::Input
-    Fluent::Plugin.register_input('plugin_test_dummy2', self)
-    helpers :storage
-    config_section :storage do
-      config_set_default :@type, 'plugin_test_dummy1'
-    end
-    def multi_workers_ready?
-      true
-    end
-  end
-  class DummyFilter < Fluent::Plugin::Filter
-    Fluent::Plugin.register_filter('plugin_test_dummy', self)
-    helpers :parser, :formatter
-    config_section :parse do
-      config_set_default :@type, 'plugin_test_dummy'
-    end
-    config_section :format do
-      config_set_default :@type, 'plugin_test_dummy'
-    end
-    def filter(tag, time, record)
-      record
-    end
-  end
-  class Dummy1Output < Fluent::Plugin::Output
-    Fluent::Plugin.register_output('plugin_test_dummy1', self)
-    def write(chunk)
-      # drop
-    end
-  end
-  class Dummy2Output < Fluent::Plugin::Output
-    Fluent::Plugin.register_output('plugin_test_dummy2', self)
-    config_section :buffer do
-      config_set_default :@type, 'plugin_test_dummy1'
-    end
-    def multi_workers_ready?
-      true
-    end
-    def write(chunk)
-      # drop
-    end
-  end
-  class Dummy1Buffer < Fluent::Plugin::Buffer
-    Fluent::Plugin.register_buffer('plugin_test_dummy1', self)
-  end
-  class Dummy2Buffer < Fluent::Plugin::Buffer
-    Fluent::Plugin.register_buffer('plugin_test_dummy2', self)
-    def multi_workers_ready?
-      false
-    end
-  end
-  class DummyParser < Fluent::Plugin::Parser
-    Fluent::Plugin.register_parser('plugin_test_dummy', self)
-  end
-  class DummyFormatter < Fluent::Plugin::Formatter
-    Fluent::Plugin.register_formatter('plugin_test_dummy', self)
-  end
-  class Dummy1Storage < Fluent::Plugin::Storage
-    Fluent::Plugin.register_storage('plugin_test_dummy1', self)
-  end
-  class Dummy2Storage < Fluent::Plugin::Storage
-    Fluent::Plugin.register_storage('plugin_test_dummy2', self)
-    def multi_workers_ready?
-      false
-    end
-  end
-  class DummyOwner < Fluent::Plugin::Base
-    include Fluent::PluginId
-    include Fluent::PluginLoggerMixin
-  end
-  class DummyEventRouter
-    def emit(tag, time, record); end
-    def emit_array(tag, array); end
-    def emit_stream(tag, es); end
-    def emit_error_event(tag, time, record, error); end
-  end
-
-  sub_test_case '#new_* methods' do
-    data(
-      input1: ['plugin_test_dummy1', Dummy1Input, :new_input],
-      input2: ['plugin_test_dummy2', Dummy2Input, :new_input],
-      filter: ['plugin_test_dummy', DummyFilter, :new_filter],
-      output1: ['plugin_test_dummy1', Dummy1Output, :new_output],
-      output2: ['plugin_test_dummy2', Dummy2Output, :new_output],
-    )
-    test 'returns plugin instances of registered plugin classes' do |(type, klass, m)|
-      instance = Fluent::Plugin.__send__(m, type)
-      assert_kind_of klass, instance
-    end
-
-    data(
-      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer],
-      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer],
-      parser: ['plugin_test_dummy', DummyParser, :new_parser],
-      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter],
-      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage],
-      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage],
-    )
-    test 'returns plugin instances of registered owned plugin classes' do |(type, klass, m)|
-      owner = DummyOwner.new
-      instance = Fluent::Plugin.__send__(m, type, parent: owner)
-      assert_kind_of klass, instance
-    end
-
-    data(
-      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],
-      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],
-      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],
-      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],
-      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],
-      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],
-      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],
-      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],
-      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],
-      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],
-      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],
-    )
-    test 'returns plugin instances which are extended by FeatureAvailabilityChecker module' do |(type, _, m, kwargs)|
-      instance = if kwargs
-                   Fluent::Plugin.__send__(m, type, **kwargs)
-                 else
-                   Fluent::Plugin.__send__(m, type)
-                 end
-      assert_kind_of Fluent::Plugin::FeatureAvailabilityChecker, instance
-    end
-  end
-
-  sub_test_case 'with default system configuration' do
-    data(
-      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],
-      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],
-      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],
-      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],
-      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],
-      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],
-      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],
-      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],
-      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],
-      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],
-      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],
-    )
-    test '#configure does not raise anything' do |(type, _, m, kwargs)|
-      instance = if kwargs
-                   Fluent::Plugin.__send__(m, type, **kwargs)
-                 else
-                   Fluent::Plugin.__send__(m, type)
-                 end
-      if instance.respond_to?(:context_router=)
-        instance.context_router = DummyEventRouter.new
-      end
-      assert_nothing_raised do
-        instance.configure(config_element())
-      end
-    end
-  end
-
-  sub_test_case 'with single worker configuration' do
-    data(
-      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],
-      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],
-      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],
-      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],
-      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],
-      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],
-      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],
-      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],
-      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],
-      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],
-      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],
-    )
-    test '#configure does not raise anything' do |(type, _, m, kwargs)|
-      instance = if kwargs
-                   Fluent::Plugin.__send__(m, type, **kwargs)
-                 else
-                   Fluent::Plugin.__send__(m, type)
-                 end
-      if instance.respond_to?(:context_router=)
-        instance.context_router = DummyEventRouter.new
-      end
-      assert_nothing_raised do
-        instance.system_config_override('workers' => 1)
-        instance.configure(config_element())
-      end
-    end
-  end
-
-  sub_test_case 'with multi workers configuration' do
-    data(
-      input1: ['plugin_test_dummy1', Dummy1Input, :new_input],
-      output1: ['plugin_test_dummy1', Dummy1Output, :new_output],
-    )
-    test '#configure raise configuration error if plugins are not ready for multi workers' do |(type, klass, new_method)|
-      conf = config_element()
-      instance = Fluent::Plugin.__send__(new_method, type)
-      if instance.respond_to?(:context_router=)
-        instance.context_router = DummyEventRouter.new
-      end
-      assert_raise Fluent::ConfigError.new("Plugin '#{type}' does not support multi workers configuration (#{klass})") do
-        instance.system_config_override('workers' => 3)
-        instance.configure(conf)
-      end
-    end
-
-    data(
-      input2: ['plugin_test_dummy2', Dummy2Input, :new_input], # with Dummy1Storage
-      filter: ['plugin_test_dummy', DummyFilter, :new_filter], # with DummyParser and DummyFormatter
-      output2: ['plugin_test_dummy2', Dummy2Output, :new_output], # with Dummy1Buffer
-    )
-    test '#configure does not raise any errors if plugins and its owned plugins are ready for multi workers' do |(type, _klass, new_method)|
-      conf = config_element()
-      instance = Fluent::Plugin.__send__(new_method, type)
-      if instance.respond_to?(:context_router=)
-        instance.context_router = DummyEventRouter.new
-      end
-      assert_nothing_raised do
-        instance.system_config_override('workers' => 3)
-        instance.configure(conf)
-      end
-    end
-
-    data(
-      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, 'storage', 'plugin_test_dummy2', Dummy2Storage],
-      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, 'buffer', 'plugin_test_dummy2', Dummy2Buffer],
-    )
-    test '#configure raise configuration error if configured owned plugins are not ready for multi workers' do |(type, _klass, new_method, subsection, subsection_type, problematic)|
-      conf = config_element('root', '', {}, [config_element(subsection, '', {'@type' => subsection_type})])
-      instance = Fluent::Plugin.__send__(new_method, type)
-      if instance.respond_to?(:context_router=)
-        instance.context_router = DummyEventRouter.new
-      end
-      assert_raise Fluent::ConfigError.new("Plugin '#{subsection_type}' does not support multi workers configuration (#{problematic})") do
-        instance.system_config_override('workers' => 3)
-        instance.configure(conf)
-      end
-    end
-  end
-end