fluentd-hubspot 0.14.14.1

data/test/test_output.rb

@@ -0,0 +1,273 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/output'
+require 'fluent/output_chain'
+require 'fluent/plugin/buffer'
+require 'timecop'
+require 'flexmock/test_unit'
+
+module FluentOutputTest
+  include Fluent
+  include FlexMock::TestCase
+
+  class BufferedOutputTest < ::Test::Unit::TestCase
+    include FluentOutputTest
+
+    class << self
+      def startup
+        $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), 'scripts'))
+        require 'fluent/plugin/out_test'
+        require 'fluent/plugin/out_test2'
+      end
+
+      def shutdown
+        $LOAD_PATH.shift
+      end
+    end
+
+    def setup
+      Fluent::Test.setup
+    end
+
+    CONFIG = %[]
+
+    def create_driver(conf=CONFIG)
+      Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
+        def write(chunk)
+          chunk.read
+        end
+      end.configure(conf)
+    end
+
+    def test_configure
+      # default
+      d = create_driver
+      assert_equal 'memory', d.instance.buffer_type
+      assert_equal 60, d.instance.flush_interval
+      assert_equal false, d.instance.disable_retry_limit
+      assert_equal 17, d.instance.retry_limit
+      assert_equal 1.0, d.instance.retry_wait
+      assert_equal nil, d.instance.max_retry_wait
+      assert_equal 1.0, d.instance.retry_wait
+      assert_equal 1, d.instance.num_threads
+      assert_equal 1, d.instance.queued_chunk_flush_interval
+
+      # max_retry_wait
+      d = create_driver(CONFIG + %[max_retry_wait 4])
+      assert_equal 4, d.instance.max_retry_wait
+
+      # disable_retry_limit
+      d = create_driver(CONFIG + %[disable_retry_limit true])
+      assert_equal true, d.instance.disable_retry_limit
+
+      #### retry_state cares it
+      # # retry_wait is converted to Float for calc_retry_wait
+      # d = create_driver(CONFIG + %[retry_wait 1s])
+      # assert_equal Float, d.instance.retry_wait.class
+    end
+
+    class FormatterInjectTestOutput < Fluent::Output
+      def initialize
+        super
+        @formatter = nil
+      end
+    end
+    def test_start
+      i = FormatterInjectTestOutput.new
+      i.configure(config_element('ROOT', '', {}, [config_element('inject', '', {'hostname_key' => "host"})]))
+      assert_nothing_raised do
+        i.start
+      end
+    end
+
+    def create_mock_driver(conf=CONFIG)
+      Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
+        attr_accessor :submit_flush_threads
+
+        def start_mock
+          @started = false
+          start
+          # ensure OutputThread to start successfully
+          submit_flush
+          sleep 0.5
+          while !@started
+            submit_flush
+            sleep 0.5
+          end
+        end
+
+        def try_flush
+          @started = true
+          @submit_flush_threads ||= {}
+          @submit_flush_threads[Thread.current] ||= 0
+          @submit_flush_threads[Thread.current] += 1
+        end
+
+        def write(chunk)
+          chunk.read
+        end
+      end.configure(conf)
+    end
+
+    def test_secondary
+      d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
+        def write(chunk)
+          chunk.read
+        end
+      end
+
+      mock(d.instance.log).warn("secondary type should be same with primary one",
+                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" })
+      d.configure(CONFIG + %[
+        <secondary>
+          type test2
+          name c0
+        </secondary>
+      ])
+
+      assert_not_nil d.instance.instance_variable_get(:@secondary).router
+    end
+
+    def test_secondary_with_no_warn_log
+      # ObjectBufferedOutput doesn't implemnt `custom_filter`
+      d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
+
+      mock(d.instance.log).warn("secondary type should be same with primary one",
+                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" }).never
+      d.configure(CONFIG + %[
+        <secondary>
+          type test2
+          name c0
+        </secondary>
+      ])
+
+      assert_not_nil d.instance.instance_variable_get(:@secondary).router
+    end
+
+    test 'BufferQueueLimitError compatibility' do
+      assert_equal Fluent::Plugin::Buffer::BufferOverflowError, Fluent::BufferQueueLimitError
+    end
+  end
+
+  class ObjectBufferedOutputTest < ::Test::Unit::TestCase
+    include FluentOutputTest
+
+    def setup
+      Fluent::Test.setup
+    end
+
+    CONFIG = %[]
+
+    def create_driver(conf=CONFIG)
+      Fluent::Test::OutputTestDriver.new(Fluent::ObjectBufferedOutput).configure(conf, true)
+    end
+
+    def test_configure
+      # default
+      d = create_driver
+      assert_equal true, d.instance.time_as_integer
+    end
+  end
+
+  class TimeSlicedOutputTest < ::Test::Unit::TestCase
+    include FluentOutputTest
+    include FlexMock::TestCase
+
+    def setup
+      Fluent::Test.setup
+      FileUtils.rm_rf(TMP_DIR)
+      FileUtils.mkdir_p(TMP_DIR)
+    end
+
+    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/time_sliced_output")
+
+    CONFIG = %[
+      buffer_path #{TMP_DIR}/foo
+      time_slice_format %Y%m%d%H
+    ]
+
+    class TimeSlicedOutputTestPlugin < Fluent::TimeSlicedOutput
+      attr_reader :written_chunk_keys, :errors_in_write
+      def initialize
+        super
+        @written_chunk_keys = []
+        @errors_in_write = []
+      end
+
+      def configure(conf)
+        super
+
+        @formatter = Fluent::Plugin.new_formatter('out_file')
+        @formatter.configure(conf)
+      end
+
+      def format(tag, time, record)
+        @formatter.format(tag, time, record)
+      end
+      def write(chunk)
+        @written_chunk_keys << chunk.key
+        true
+      rescue => e
+        @errors_in_write << e
+      end
+    end
+
+    def create_driver(conf=CONFIG)
+      Fluent::Test::TimeSlicedOutputTestDriver.new(TimeSlicedOutputTestPlugin).configure(conf, true)
+    end
+
+    data(:none => '',
+         :utc => "utc",
+         :localtime => 'localtime',
+         :timezone => 'timezone +0000')
+    test 'configure with timezone related parameters' do |param|
+      assert_nothing_raised {
+        create_driver(CONFIG + param)
+      }
+    end
+
+    sub_test_case "test emit" do
+      setup do
+        @time = Time.parse("2011-01-02 13:14:15 UTC")
+        Timecop.freeze(@time)
+      end
+
+      teardown do
+        Timecop.return
+      end
+
+      test "emit with invalid event" do
+        d = create_driver
+        d.instance.start
+        d.instance.after_start
+        assert_raise ArgumentError, "time must be a Fluent::EventTime (or Integer)" do
+          d.instance.emit_events('test', OneEventStream.new('string', 10))
+        end
+      end
+
+      test "plugin can get key of chunk in #write" do
+        d = create_driver
+        d.instance.start
+        d.instance.after_start
+        d.instance.emit_events('test', OneEventStream.new(event_time("2016-11-08 17:44:30 +0900"), {"message" => "yay"}))
+        d.instance.force_flush
+        waiting(10) do
+          sleep 0.1 until d.instance.written_chunk_keys.size == 1
+        end
+        assert_equal [], d.instance.errors_in_write
+        assert_equal ["2016110808"], d.instance.written_chunk_keys # default timezone is UTC
+      end
+
+      test "check formatted time compatibility with utc. Should Z, not +00:00" do
+        d = create_driver(CONFIG + %[
+          utc
+          include_time_key
+        ])
+        time = Time.parse("2016-11-08 12:00:00 UTC").to_i
+        d.emit({"a" => 1}, time)
+        d.expect_format %[2016-11-08T12:00:00Z\ttest\t{"a":1,"time":"2016-11-08T12:00:00Z"}\n]
+        d.run
+      end
+    end
+  end
+end

data/test/test_plugin.rb

@@ -0,0 +1,251 @@
+require_relative 'helper'
+
+require 'fluent/plugin'
+require 'fluent/plugin/input'
+require 'fluent/plugin/filter'
+require 'fluent/plugin/output'
+require 'fluent/plugin/buffer'
+require 'fluent/plugin/parser'
+require 'fluent/plugin/formatter'
+require 'fluent/plugin/storage'
+
+class PluginTest < Test::Unit::TestCase
+  class Dummy1Input < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('plugin_test_dummy1', self)
+  end
+  class Dummy2Input < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('plugin_test_dummy2', self)
+    helpers :storage
+    config_section :storage do
+      config_set_default :@type, 'plugin_test_dummy1'
+    end
+    def multi_workers_ready?
+      true
+    end
+  end
+  class DummyFilter < Fluent::Plugin::Filter
+    Fluent::Plugin.register_filter('plugin_test_dummy', self)
+    helpers :parser, :formatter
+    config_section :parse do
+      config_set_default :@type, 'plugin_test_dummy'
+    end
+    config_section :format do
+      config_set_default :@type, 'plugin_test_dummy'
+    end
+    def filter(tag, time, record)
+      record
+    end
+  end
+  class Dummy1Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('plugin_test_dummy1', self)
+    def write(chunk)
+      # drop
+    end
+  end
+  class Dummy2Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('plugin_test_dummy2', self)
+    config_section :buffer do
+      config_set_default :@type, 'plugin_test_dummy1'
+    end
+    def multi_workers_ready?
+      true
+    end
+    def write(chunk)
+      # drop
+    end
+  end
+  class Dummy1Buffer < Fluent::Plugin::Buffer
+    Fluent::Plugin.register_buffer('plugin_test_dummy1', self)
+  end
+  class Dummy2Buffer < Fluent::Plugin::Buffer
+    Fluent::Plugin.register_buffer('plugin_test_dummy2', self)
+    def multi_workers_ready?
+      false
+    end
+  end
+  class DummyParser < Fluent::Plugin::Parser
+    Fluent::Plugin.register_parser('plugin_test_dummy', self)
+  end
+  class DummyFormatter < Fluent::Plugin::Formatter
+    Fluent::Plugin.register_formatter('plugin_test_dummy', self)
+  end
+  class Dummy1Storage < Fluent::Plugin::Storage
+    Fluent::Plugin.register_storage('plugin_test_dummy1', self)
+  end
+  class Dummy2Storage < Fluent::Plugin::Storage
+    Fluent::Plugin.register_storage('plugin_test_dummy2', self)
+    def multi_workers_ready?
+      false
+    end
+  end
+  class DummyOwner < Fluent::Plugin::Base
+    include Fluent::PluginId
+    include Fluent::PluginLoggerMixin
+  end
+  class DummyEventRouter
+    def emit(tag, time, record); end
+    def emit_array(tag, array); end
+    def emit_stream(tag, es); end
+    def emit_error_event(tag, time, record, error); end
+  end
+
+  sub_test_case '#new_* methods' do
+    data(
+      input1: ['plugin_test_dummy1', Dummy1Input, :new_input],
+      input2: ['plugin_test_dummy2', Dummy2Input, :new_input],
+      filter: ['plugin_test_dummy', DummyFilter, :new_filter],
+      output1: ['plugin_test_dummy1', Dummy1Output, :new_output],
+      output2: ['plugin_test_dummy2', Dummy2Output, :new_output],
+    )
+    test 'retruns plugin instances of registered plugin classes' do |(type, klass, m)|
+      instance = Fluent::Plugin.__send__(m, type)
+      assert_kind_of klass, instance
+    end
+
+    data(
+      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer],
+      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer],
+      parser: ['plugin_test_dummy', DummyParser, :new_parser],
+      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter],
+      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage],
+      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage],
+    )
+    test 'returns plugin instances of registered owned plugin classes' do |(type, klass, m)|
+      owner = DummyOwner.new
+      instance = Fluent::Plugin.__send__(m, type, parent: owner)
+      assert_kind_of klass, instance
+    end
+
+    data(
+      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],
+      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],
+      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],
+      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],
+      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],
+      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],
+      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],
+      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],
+      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],
+      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],
+      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],
+    )
+    test 'returns plugin instances which are extended by FeatureAvailabilityChecker module' do |(type, _, m, kwargs)|
+      instance = if kwargs
+                   Fluent::Plugin.__send__(m, type, **kwargs)
+                 else
+                   Fluent::Plugin.__send__(m, type)
+                 end
+      assert_kind_of Fluent::Plugin::FeatureAvailabilityChecker, instance
+    end
+  end
+
+  sub_test_case 'with default system configuration' do
+    data(
+      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],
+      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],
+      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],
+      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],
+      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],
+      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],
+      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],
+      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],
+      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],
+      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],
+      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],
+    )
+    test '#configure does not raise anything' do |(type, _, m, kwargs)|
+      instance = if kwargs
+                   Fluent::Plugin.__send__(m, type, **kwargs)
+                 else
+                   Fluent::Plugin.__send__(m, type)
+                 end
+      if instance.respond_to?(:context_router=)
+        instance.context_router = DummyEventRouter.new
+      end
+      assert_nothing_raised do
+        instance.configure(config_element())
+      end
+    end
+  end
+
+  sub_test_case 'with single worker configuration' do
+    data(
+      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],
+      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],
+      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],
+      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],
+      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],
+      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],
+      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],
+      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],
+      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],
+      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],
+      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],
+    )
+    test '#configure does not raise anything' do |(type, _, m, kwargs)|
+      instance = if kwargs
+                   Fluent::Plugin.__send__(m, type, **kwargs)
+                 else
+                   Fluent::Plugin.__send__(m, type)
+                 end
+      if instance.respond_to?(:context_router=)
+        instance.context_router = DummyEventRouter.new
+      end
+      assert_nothing_raised do
+        instance.system_config_override('workers' => 1)
+        instance.configure(config_element())
+      end
+    end
+  end
+
+  sub_test_case 'with multi workers configuration' do
+    data(
+      input1: ['plugin_test_dummy1', Dummy1Input, :new_input],
+      output1: ['plugin_test_dummy1', Dummy1Output, :new_output],
+    )
+    test '#configure raise configuration error if plugins are not ready for multi workers' do |(type, klass, new_method)|
+      conf = config_element()
+      instance = Fluent::Plugin.__send__(new_method, type)
+      if instance.respond_to?(:context_router=)
+        instance.context_router = DummyEventRouter.new
+      end
+      assert_raise Fluent::ConfigError.new("Plugin '#{type}' does not support multi workers configuration (#{klass})") do
+        instance.system_config_override('workers' => 3)
+        instance.configure(conf)
+      end
+    end
+
+    data(
+      input2: ['plugin_test_dummy2', Dummy2Input, :new_input], # with Dummy1Storage
+      filter: ['plugin_test_dummy', DummyFilter, :new_filter], # with DummyParser and DummyFormatter
+      output2: ['plugin_test_dummy2', Dummy2Output, :new_output], # with Dummy1Buffer
+    )
+    test '#configure does not raise any errors if plugins and its owned plugins are ready for multi workers' do |(type, _klass, new_method)|
+      conf = config_element()
+      instance = Fluent::Plugin.__send__(new_method, type)
+      if instance.respond_to?(:context_router=)
+        instance.context_router = DummyEventRouter.new
+      end
+      assert_nothing_raised do
+        instance.system_config_override('workers' => 3)
+        instance.configure(conf)
+      end
+    end
+
+    data(
+      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, 'storage', 'plugin_test_dummy2', Dummy2Storage],
+      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, 'buffer', 'plugin_test_dummy2', Dummy2Buffer],
+    )
+    test '#configure raise configuration error if configured owned plugins are not ready for multi workers' do |(type, _klass, new_method, subsection, subsection_type, problematic)|
+      conf = config_element('root', '', {}, [config_element(subsection, '', {'@type' => subsection_type})])
+      instance = Fluent::Plugin.__send__(new_method, type)
+      if instance.respond_to?(:context_router=)
+        instance.context_router = DummyEventRouter.new
+      end
+      assert_raise Fluent::ConfigError.new("Plugin '#{subsection_type}' does not support multi workers configuration (#{problematic})") do
+        instance.system_config_override('workers' => 3)
+        instance.configure(conf)
+      end
+    end
+  end
+end