fluentd 0.14.4-x86-mingw32

data/test/plugin_helper/test_compat_parameters.rb

@@ -0,0 +1,242 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/compat_parameters'
+require 'fluent/plugin/base'
+
+require 'time'
+
+class CompatParameterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @i = nil
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+  end
+
+  class DummyO0 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: '')
+      super
+    end
+    def write(chunk); end # dummy
+  end
+  class DummyO1 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'time')
+      super
+    end
+    def write(chunk); end # dummy
+  end
+  class DummyO2 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'time')
+      super
+    end
+    def write(chunk); end # dummy
+  end
+  class DummyO3 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'tag')
+      super
+    end
+    def write(chunk); end # dummy
+  end
+  class DummyO4 < Fluent::Plugin::Output
+    helpers :compat_parameters, :inject, :formatter
+    attr_reader :f
+    def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject, :formatter, default_chunk_key: 'tag')
+      super
+    end
+    def start
+      super
+      @f = formatter_create()
+    end
+    def write(chunk); end # dummy
+  end
+
+  sub_test_case 'output plugins which does not have default chunk key' do
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'num_threads' => 8,
+        'flush_interval' => '10s',
+        'buffer_chunk_limit' => '8m',
+        'buffer_queue_limit' => '1024',
+        'flush_at_shutdown' => 'yes',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO0.new
+      @i.configure(conf)
+
+      assert_equal 'memory', @i.buffer_config[:@type]
+      assert_equal [], @i.buffer_config.chunk_keys
+      assert_equal 8, @i.buffer_config.flush_thread_count
+      assert_equal 10, @i.buffer_config.flush_interval
+      assert @i.buffer_config.flush_at_shutdown
+
+      assert_equal 8*1024*1024, @i.buffer.chunk_limit_size
+      assert_equal 1024, @i.buffer.queue_length_limit
+    end
+  end
+
+  sub_test_case 'output plugins which has default chunk key: time' do
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => '/tmp/mybuffer',
+        'disable_retry_limit' => 'yes',
+        'max_retry_wait' => '1h',
+        'buffer_queue_full_action' => 'block',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO1.new
+      @i.configure(conf)
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 24*60*60, @i.buffer_config.timekey
+      assert @i.buffer_config.retry_forever
+      assert_equal 60*60, @i.buffer_config.retry_max_interval
+      assert_equal :block, @i.buffer_config.overflow_action
+
+      assert !@i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      assert_equal '/tmp/mybuffer/buffer.*.log', @i.buffer.path
+    end
+  end
+
+  sub_test_case 'output plugins which does not have default chunk key' do
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => '/tmp/mybuffer',
+        'time_slice_format' => '%Y%m%d%H',
+        'time_slice_wait' => '10',
+        'retry_limit' => '1024',
+        'buffer_queue_full_action' => 'drop_oldest_chunk',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO2.new
+      @i.configure(conf)
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 60*60, @i.buffer_config.timekey
+      assert_equal 10, @i.buffer_config.timekey_wait
+      assert_equal 1024, @i.buffer_config.retry_max_times
+      assert_equal :drop_oldest_chunk, @i.buffer_config.overflow_action
+
+      assert @i.chunk_key_time
+      assert !@i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      assert_equal '/tmp/mybuffer/buffer.*.log', @i.buffer.path
+    end
+  end
+
+  sub_test_case 'output plugins which has default chunk key: tag' do
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'memory',
+        'num_threads' => '10',
+        'flush_interval' => '10s',
+        'try_flush_interval' => '0.1',
+        'queued_chunk_flush_interval' => '0.5',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO3.new
+      @i.configure(conf)
+
+      assert_equal 'memory', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      assert_equal 10, @i.buffer_config.flush_interval
+      assert_equal 0.1, @i.buffer_config.flush_thread_interval
+      assert_equal 0.5, @i.buffer_config.flush_thread_burst_interval
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+    end
+  end
+
+  sub_test_case 'output plugins which has default chunk key: tag, and enables inject and formatter' do
+    test 'plugin helper converts parameters into plugin configuration parameters for all of buffer, inject and formatter' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => File.expand_path('../../tmp/compat_parameters/mybuffer.*.log', __FILE__),
+        'num_threads' => '10',
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'include_time_key' => 'true', # default time_key 'time' and default time format (iso8601: 2016-06-24T15:57:38) at localtime
+        'include_tag_key' => 'yes', # default tag_key 'tag'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO4.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      formatter = @i.f
+      assert{ formatter.is_a? Fluent::Plugin::LabeledTSVFormatter }
+      assert_equal ',', @i.f.delimiter
+      assert_equal '%', @i.f.label_delimiter
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      t = event_time('2016-06-24 16:05:01') # localtime
+      iso8601str = Time.at(t.to_i).iso8601
+      formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+    end
+
+    test 'plugin helper setups time injecting as unix time (integer from epoch)' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => File.expand_path('../../tmp/compat_parameters/mybuffer.*.log', __FILE__),
+        'num_threads' => '10',
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'include_time_key' => 'true', # default time_key 'time' and default time format (iso8601: 2016-06-24T15:57:38) at localtime
+        'include_tag_key' => 'yes', # default tag_key 'tag'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO4.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      formatter = @i.f
+      assert{ formatter.is_a? Fluent::Plugin::LabeledTSVFormatter }
+      assert_equal ',', @i.f.delimiter
+      assert_equal '%', @i.f.label_delimiter
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      t = event_time('2016-06-24 16:05:01') # localtime
+      iso8601str = Time.at(t.to_i).iso8601
+      formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+    end
+  end
+end

data/test/plugin_helper/test_event_emitter.rb

@@ -0,0 +1,51 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/event_emitter'
+require 'fluent/plugin/base'
+
+class EventEmitterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  class Dummy0 < Fluent::Plugin::TestBase
+  end
+
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :event_emitter
+  end
+
+  test 'can be instantiated to be able to emit with router' do
+    d0 = Dummy0.new
+    assert d0.respond_to?(:has_router?)
+    assert !d0.has_router?
+    assert !d0.respond_to?(:router)
+
+    d1 = Dummy.new
+    assert d1.respond_to?(:has_router?)
+    assert d1.has_router?
+    assert d1.respond_to?(:router)
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can be configured with valid router' do
+    d1 = Dummy.new
+    assert d1.has_router?
+    assert_nil d1.router
+
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+
+    assert d1.router
+
+    d1.shutdown
+
+    assert d1.router
+
+    d1.close
+
+    assert_nil d1.router
+
+    d1.terminate
+  end
+end

data/test/plugin_helper/test_event_loop.rb

@@ -0,0 +1,52 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/event_loop'
+require 'fluent/plugin/base'
+
+class EventLoopTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :event_loop
+    def configure(conf)
+      super
+      @_event_loop_run_timeout = 0.1
+    end
+  end
+
+  test 'can be instantiated to be able to create event loop' do
+    d1 = Dummy.new
+    assert d1.respond_to?(:event_loop_attach)
+    assert d1.respond_to?(:event_loop_running?)
+    assert d1.respond_to?(:_event_loop)
+    assert d1._event_loop
+    assert !d1.event_loop_running?
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'can run event loop by start, stop by shutdown/close and clear by terminate' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.event_loop_running?
+
+    d1.start
+    d1.event_loop_wait_until_start
+
+    assert d1.event_loop_running?
+    assert_equal 1, d1._event_loop.watchers.size
+
+    d1.shutdown
+    d1.close
+
+    assert !d1.event_loop_running?
+
+    d1.terminate
+
+    assert_nil d1._event_loop
+  end
+end

data/test/plugin_helper/test_formatter.rb

@@ -0,0 +1,252 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/formatter'
+require 'fluent/plugin/base'
+
+class FormatterHelperTest < Test::Unit::TestCase
+  class ExampleFormatter < Fluent::Plugin::Formatter
+    Fluent::Plugin.register_formatter('example', self)
+    def format(tag, time, record)
+      "#{tag},#{time.to_i},#{record.keys.sort.join(',')}" # hey, you miss values! :P
+    end
+  end
+  class Example2Formatter < Fluent::Plugin::Formatter
+    Fluent::Plugin.register_formatter('example2', self)
+    def format(tag, time, record)
+      "#{tag},#{time.to_i},#{record.values.sort.join(',')}" # key...
+    end
+  end
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :formatter
+  end
+
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :formatter
+    config_section :format do
+      config_set_default :@type, 'example2'
+    end
+  end
+
+  setup do
+    @d = nil
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'can be initialized without any formatters at first' do
+    d = Dummy.new
+    assert_equal 0, d._formatters.size
+  end
+
+  test 'can override default configuration parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    assert_equal [], d.formatter_configs
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('format', '', {}, [])]))
+    assert_raise NoMethodError do
+      d.format
+    end
+    assert_equal 1, d.formatter_configs.size
+    assert_equal 'example2', d.formatter_configs.first[:@type]
+  end
+
+  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.formatter_create(conf: config_element('format', '', {'@type' => 'example'}), default_type: 'example2')
+    assert{ i.is_a?(ExampleFormatter) }
+
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.formatter_create(conf: nil, default_type: 'example2')
+    assert{ i.is_a?(Example2Formatter) }
+  end
+
+  test 'raises config error if config section is specified, but @type is not specified' do
+    d = Dummy2.new
+    d.configure(config_element())
+    assert_raise Fluent::ConfigError.new("@type is required in <format>") do
+      d.formatter_create(conf: config_element('format', '', {}), default_type: 'example2')
+    end
+  end
+
+  test 'can be configured without format sections' do
+    d = Dummy.new
+    assert_nothing_raised do
+      d.configure(config_element())
+    end
+    assert_equal 0, d._formatters.size
+  end
+
+  test 'can be configured with a format section' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', '', {'@type' => 'example'})
+      ])
+    assert_nothing_raised do
+      d.configure(conf)
+    end
+    assert_equal 1, d._formatters.size
+    assert{ d._formatters.values.all?{ |formatter| !formatter.started? } }
+  end
+
+  test 'can be configured with 2 or more format sections with different usages with each other' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', 'default', {'@type' => 'example'}),
+        config_element('format', 'extra', {'@type' => 'example2'}),
+      ])
+    assert_nothing_raised do
+      d.configure(conf)
+    end
+    assert_equal 2, d._formatters.size
+    assert{ d._formatters.values.all?{ |formatter| !formatter.started? } }
+  end
+
+  test 'cannot be configured with 2 format sections with same usage' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', 'default', {'@type' => 'example'}),
+        config_element('format', 'extra', {'@type' => 'example2'}),
+        config_element('format', 'extra', {'@type' => 'example2'}),
+      ])
+    assert_raises Fluent::ConfigError do
+      d.configure(conf)
+    end
+  end
+
+  test 'creates a format plugin instance which is already configured without usage' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', '', {'@type' => 'example'})
+      ])
+    d.configure(conf)
+    d.start
+
+    formatter = d.formatter_create
+    assert{ formatter.is_a? ExampleFormatter }
+    assert formatter.started?
+  end
+
+  test 'creates a formatter plugin instance which is already configured with usage' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', 'mydata', {'@type' => 'example'})
+      ])
+    d.configure(conf)
+    d.start
+
+    formatter = d.formatter_create(usage: 'mydata')
+    assert{ formatter.is_a? ExampleFormatter }
+    assert formatter.started?
+  end
+
+  test 'creates a formatter plugin without configurations' do
+    @d = d = Dummy.new
+    d.configure(config_element())
+    d.start
+
+    formatter = d.formatter_create(usage: 'mydata', type: 'example', conf: config_element('format', 'mydata'))
+    assert{ formatter.is_a? ExampleFormatter }
+    assert formatter.started?
+  end
+
+  test 'creates 2 or more formatter plugin instances' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', 'mydata', {'@type' => 'example'}),
+        config_element('format', 'secret', {'@type' => 'example2'})
+      ])
+    d.configure(conf)
+    d.start
+
+    p1 = d.formatter_create(usage: 'mydata')
+    p2 = d.formatter_create(usage: 'secret')
+    assert{ p1.is_a? ExampleFormatter }
+    assert p1.started?
+    assert{ p2.is_a? Example2Formatter }
+    assert p2.started?
+  end
+
+  test 'calls lifecycle methods for all plugin instances via owner plugin' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [ config_element('format', '', {'@type' => 'example'}), config_element('format', 'e2', {'@type' => 'example'}) ])
+    d.configure(conf)
+    d.start
+
+    i1 = d.formatter_create(usage: '')
+    i2 = d.formatter_create(usage: 'e2')
+    i3 = d.formatter_create(usage: 'e3', type: 'example2')
+
+    assert i1.started?
+    assert i2.started?
+    assert i3.started?
+
+    assert !i1.stopped?
+    assert !i2.stopped?
+    assert !i3.stopped?
+
+    d.stop
+
+    assert i1.stopped?
+    assert i2.stopped?
+    assert i3.stopped?
+
+    assert !i1.before_shutdown?
+    assert !i2.before_shutdown?
+    assert !i3.before_shutdown?
+
+    d.before_shutdown
+
+    assert i1.before_shutdown?
+    assert i2.before_shutdown?
+    assert i3.before_shutdown?
+
+    assert !i1.shutdown?
+    assert !i2.shutdown?
+    assert !i3.shutdown?
+
+    d.shutdown
+
+    assert i1.shutdown?
+    assert i2.shutdown?
+    assert i3.shutdown?
+
+    assert !i1.after_shutdown?
+    assert !i2.after_shutdown?
+    assert !i3.after_shutdown?
+
+    d.after_shutdown
+
+    assert i1.after_shutdown?
+    assert i2.after_shutdown?
+    assert i3.after_shutdown?
+
+    assert !i1.closed?
+    assert !i2.closed?
+    assert !i3.closed?
+
+    d.close
+
+    assert i1.closed?
+    assert i2.closed?
+    assert i3.closed?
+
+    assert !i1.terminated?
+    assert !i2.terminated?
+    assert !i3.terminated?
+
+    d.terminate
+
+    assert i1.terminated?
+    assert i2.terminated?
+    assert i3.terminated?
+  end
+end