fluentd 0.12.40 → 1.6.2

data/test/plugin_helper/test_compat_parameters.rb

@@ -0,0 +1,353 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/compat_parameters'
+require 'fluent/plugin/input'
+require 'fluent/plugin/output'
+require 'fluent/time'
+
+require 'time'
+
+class CompatParameterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @i = nil
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+  end
+
+  class DummyI0 < Fluent::Plugin::Input
+    helpers :compat_parameters, :parser, :extract
+    attr_reader :parser
+    def configure(conf)
+      compat_parameters_convert(conf, :extract, :parser)
+      super
+    end
+    def start
+      super
+      @parser = parser_create
+    end
+    def produce_events(input_data)
+      emit_events = [] # tag, time, record
+      @parser.parse(input_data) do |time, record|
+        tag = extract_tag_from_record(record) || 'dummy_tag'
+        emit_events << [tag, time, record]
+      end
+      emit_events
+    end
+  end
+  class DummyO0 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: '')
+      super
+    end
+    def write(chunk); end # dummy
+  end
+  class DummyO1 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'time')
+      super
+    end
+    def write(chunk); end # dummy
+  end
+  class DummyO2 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'time')
+      super
+    end
+    def write(chunk); end # dummy
+  end
+  class DummyO3 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'tag')
+      super
+    end
+    def write(chunk); end # dummy
+  end
+  class DummyO4 < Fluent::Plugin::Output
+    helpers :compat_parameters, :inject, :formatter
+    attr_reader :f
+    def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject, :formatter, default_chunk_key: 'tag')
+      super
+    end
+    def start
+      super
+      @f = formatter_create()
+    end
+    def write(chunk); end # dummy
+  end
+
+  sub_test_case 'output plugins which does not have default chunk key' do
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'num_threads' => 8,
+        'flush_interval' => '10s',
+        'buffer_chunk_limit' => '8m',
+        'buffer_queue_limit' => '1024',
+        'flush_at_shutdown' => 'yes',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO0.new
+      @i.configure(conf)
+
+      assert_equal 'memory', @i.buffer_config[:@type]
+      assert_equal [], @i.buffer_config.chunk_keys
+      assert_equal 8, @i.buffer_config.flush_thread_count
+      assert_equal 10, @i.buffer_config.flush_interval
+      assert_equal :default, @i.buffer_config.flush_mode
+      assert @i.buffer_config.flush_at_shutdown
+
+      assert_equal 8*1024*1024, @i.buffer.chunk_limit_size
+      assert_equal 1024, @i.buffer.queue_limit_length
+    end
+  end
+
+  sub_test_case 'output plugins which has default chunk key: time' do
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => '/tmp/mybuffer',
+        'disable_retry_limit' => 'yes',
+        'max_retry_wait' => '1h',
+        'buffer_queue_full_action' => 'block',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO1.new
+      @i.configure(conf)
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 24*60*60, @i.buffer_config.timekey
+      assert @i.buffer_config.retry_forever
+      assert_equal 60*60, @i.buffer_config.retry_max_interval
+      assert_equal :block, @i.buffer_config.overflow_action
+      assert_equal :default, @i.buffer_config.flush_mode
+
+      assert !@i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      assert_equal '/tmp/mybuffer/buffer.*.log', @i.buffer.path
+    end
+  end
+
+  sub_test_case 'output plugins which does not have default chunk key' do
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => '/tmp/mybuffer',
+        'time_slice_format' => '%Y%m%d%H',
+        'time_slice_wait' => '10',
+        'retry_limit' => '1024',
+        'buffer_queue_full_action' => 'drop_oldest_chunk',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO2.new
+      @i.configure(conf)
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 60*60, @i.buffer_config.timekey
+      assert_equal 10, @i.buffer_config.timekey_wait
+      assert_equal 1024, @i.buffer_config.retry_max_times
+      assert_equal :drop_oldest_chunk, @i.buffer_config.overflow_action
+
+      assert @i.chunk_key_time
+      assert !@i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      assert_equal '/tmp/mybuffer/buffer.*.log', @i.buffer.path
+    end
+  end
+
+  sub_test_case 'output plugins which has default chunk key: tag' do
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'memory',
+        'num_threads' => '10',
+        'flush_interval' => '10s',
+        'try_flush_interval' => '0.1',
+        'queued_chunk_flush_interval' => '0.5',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO3.new
+      @i.configure(conf)
+
+      assert_equal 'memory', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      assert_equal 10, @i.buffer_config.flush_interval
+      assert_equal 0.1, @i.buffer_config.flush_thread_interval
+      assert_equal 0.5, @i.buffer_config.flush_thread_burst_interval
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+    end
+  end
+
+  sub_test_case 'output plugins which has default chunk key: tag, and enables inject and formatter' do
+    test 'plugin helper converts parameters into plugin configuration parameters for all of buffer, inject and formatter' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => File.expand_path('../../tmp/compat_parameters/mybuffer.*.log', __FILE__),
+        'num_threads' => '10',
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'include_time_key' => 'true', # default time_key 'time' and default time format (iso8601: 2016-06-24T15:57:38) at localtime
+        'include_tag_key' => 'yes', # default tag_key 'tag'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO4.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      formatter = @i.f
+      assert{ formatter.is_a? Fluent::Plugin::LabeledTSVFormatter }
+      assert_equal ',', @i.f.delimiter
+      assert_equal '%', @i.f.label_delimiter
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      t = event_time('2016-06-24 16:05:01') # localtime
+      iso8601str = Time.at(t.to_i).iso8601
+      formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+    end
+
+    test 'plugin helper setups time injecting as unix time (integer from epoch)' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => File.expand_path('../../tmp/compat_parameters/mybuffer.*.log', __FILE__),
+        'num_threads' => '10',
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'include_time_key' => 'true', # default time_key 'time' and default time format (iso8601: 2016-06-24T15:57:38) at localtime
+        'include_tag_key' => 'yes', # default tag_key 'tag'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO4.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      formatter = @i.f
+      assert{ formatter.is_a? Fluent::Plugin::LabeledTSVFormatter }
+      assert_equal ',', @i.f.delimiter
+      assert_equal '%', @i.f.label_delimiter
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      t = event_time('2016-06-24 16:05:01') # localtime
+      iso8601str = Time.at(t.to_i).iso8601
+      formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+    end
+  end
+
+  sub_test_case 'input plugins' do
+    test 'plugin helper converts parameters into plugin configuration parameters for extract and parser' do
+      hash = {
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'tag_key' => 't2',
+        'time_key' => 't',
+        'time_format' => '%Y-%m-%d.%H:%M:%S.%N',
+        'utc' => 'yes',
+        'types' => 'A integer|B string|C bool',
+        'types_delimiter' => '|',
+        'types_label_delimiter' => ' ',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyI0.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      parser = @i.parser
+      assert{ parser.is_a? Fluent::Plugin::LabeledTSVParser }
+      assert_equal ',', parser.delimiter
+      assert_equal '%', parser.label_delimiter
+
+      events = @i.produce_events("A%1,B%x,C%true,t2%mytag,t%2016-10-20.03:50:11.987654321")
+      assert_equal 1, events.size
+
+      tag, time, record = events.first
+      assert_equal 'mytag', tag
+      assert_equal_event_time event_time("2016-10-20 03:50:11.987654321 +0000"), time
+      assert_equal 3, record.keys.size
+      assert_equal ['A','B','C'], record.keys.sort
+      assert_equal 1, record['A']
+      assert_equal 'x', record['B']
+      assert_equal true, record['C']
+    end
+
+    test 'plugin helper converts parameters into plugin configuration parameters for extract and parser, using numeric time' do
+      hash = {
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'tag_key' => 't2',
+        'time_key' => 't',
+        'time_type' => 'float',
+        'localtime' => 'yes',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyI0.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      parser = @i.parser
+      assert{ parser.is_a? Fluent::Plugin::LabeledTSVParser }
+      assert_equal ',', parser.delimiter
+      assert_equal '%', parser.label_delimiter
+    end
+
+    test 'plugin helper setups time extraction as unix time (integer from epoch)' do
+      # TODO:
+    end
+  end
+
+  sub_test_case 'parser plugins' do
+    test 'syslog parser parameters' do
+      hash = {
+        'format' => 'syslog',
+        'message_format' => 'rfc5424',
+        'with_priority' => 'true',
+        'rfc5424_time_format' => '%Y'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyI0.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      parser = @i.parser
+      assert_kind_of(Fluent::Plugin::SyslogParser, parser)
+      assert_equal :rfc5424, parser.message_format
+      assert_equal true, parser.with_priority
+      assert_equal '%Y', parser.rfc5424_time_format
+    end
+  end
+end

data/test/plugin_helper/test_event_emitter.rb

@@ -0,0 +1,51 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/event_emitter'
+require 'fluent/plugin/base'
+
+class EventEmitterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  class Dummy0 < Fluent::Plugin::TestBase
+  end
+
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :event_emitter
+  end
+
+  test 'can be instantiated to be able to emit with router' do
+    d0 = Dummy0.new
+    assert d0.respond_to?(:has_router?)
+    assert !d0.has_router?
+    assert !d0.respond_to?(:router)
+
+    d1 = Dummy.new
+    assert d1.respond_to?(:has_router?)
+    assert d1.has_router?
+    assert d1.respond_to?(:router)
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can be configured with valid router' do
+    d1 = Dummy.new
+    assert d1.has_router?
+    assert_nil d1.router
+
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+
+    assert d1.router
+
+    d1.shutdown
+
+    assert d1.router
+
+    d1.close
+
+    assert_nil d1.router
+
+    d1.terminate
+  end
+end

data/test/plugin_helper/test_event_loop.rb

@@ -0,0 +1,52 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/event_loop'
+require 'fluent/plugin/base'
+
+class EventLoopTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :event_loop
+    def configure(conf)
+      super
+      @_event_loop_run_timeout = 0.1
+    end
+  end
+
+  test 'can be instantiated to be able to create event loop' do
+    d1 = Dummy.new
+    assert d1.respond_to?(:event_loop_attach)
+    assert d1.respond_to?(:event_loop_running?)
+    assert d1.respond_to?(:_event_loop)
+    assert d1._event_loop
+    assert !d1.event_loop_running?
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'can run event loop by start, stop by shutdown/close and clear by terminate' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.event_loop_running?
+
+    d1.start
+    d1.event_loop_wait_until_start
+
+    assert d1.event_loop_running?
+    assert_equal 1, d1._event_loop.watchers.size
+
+    d1.shutdown
+    d1.close
+
+    assert !d1.event_loop_running?
+
+    d1.terminate
+
+    assert_nil d1._event_loop
+  end
+end

data/test/plugin_helper/test_extract.rb

@@ -0,0 +1,194 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/extract'
+require 'fluent/time'
+
+class ExtractHelperTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :extract
+  end
+
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :extract
+    config_section :extract do
+      config_set_default :tag_key, 'tag2'
+    end
+  end
+
+  def config_extract_section(hash = {})
+    config_element('ROOT', '', {}, [config_element('extract', '', hash)])
+  end
+
+  setup do
+    Fluent::Test.setup
+    @d = Dummy.new
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'can override default parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    d.configure(config_element())
+    assert_nil d.extract_config
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('extract')]))
+    assert d.extract_config
+    assert_equal 'tag2', d.extract_config.tag_key
+  end
+
+  test 'returns nil in default' do
+    @d.configure(config_extract_section())
+    @d.start
+    assert_nil @d.instance_eval{ @_extract_tag_key }
+    assert_nil @d.instance_eval{ @_extract_time_key }
+    assert_nil @d.instance_eval{ @_extract_time_parser }
+
+    record = {"key1" => "value1", "key2" => 2, "tag" => "yay", "time" => Time.now.to_i}
+
+    assert_nil @d.extract_tag_from_record(record)
+    assert_nil @d.extract_time_from_record(record)
+  end
+
+  test 'can be configured as specified' do
+    @d.configure(config_extract_section(
+        "tag_key" => "tag",
+        "time_key" => "time",
+        "time_type" => "unixtime",
+    ))
+
+    assert_equal "tag", @d.instance_eval{ @_extract_tag_key }
+    assert_equal "time", @d.instance_eval{ @_extract_time_key }
+    assert_equal :unixtime, @d.instance_eval{ @extract_config.time_type }
+    assert_not_nil @d.instance_eval{ @_extract_time_parser }
+  end
+
+  sub_test_case 'extract_tag_from_record' do
+    test 'returns tag string from specified tag_key field' do
+      @d.configure(config_extract_section("tag_key" => "tag"))
+      @d.start
+      @d.after_start
+
+      tag = @d.extract_tag_from_record({"tag" => "tag.test.code", "message" => "yay!"})
+      assert_equal "tag.test.code", tag
+    end
+
+    test 'returns tag as string by stringifying values from specified key' do
+      @d.configure(config_extract_section("tag_key" => "tag"))
+      @d.start
+      @d.after_start
+
+      tag = @d.extract_tag_from_record({"tag" => 100, "message" => "yay!"})
+      assert_equal "100", tag
+    end
+  end
+
+  sub_test_case 'extract_time_from_record' do
+    test 'returns EventTime object from specified time_key field, parsed as float in default' do
+      @d.configure(config_extract_section("time_key" => "t"))
+      @d.start
+      @d.after_start
+
+      # 1473135272 => 2016-09-06 04:14:32 UTC
+      t = @d.extract_time_from_record({"t" => 1473135272.5, "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 500_000_000), t)
+
+      t = @d.extract_time_from_record({"t" => "1473135272.5", "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 500_000_000), t)
+    end
+
+    test 'returns EventTime object, parsed as unixtime when configured so' do
+      @d.configure(config_extract_section("time_key" => "t", "time_type" => "unixtime"))
+      @d.start
+      @d.after_start
+
+      t = @d.extract_time_from_record({"t" => 1473135272, "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)
+
+      t = @d.extract_time_from_record({"t" => "1473135272", "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)
+
+      t = @d.extract_time_from_record({"t" => 1473135272.5, "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)
+    end
+
+    test 'returns EventTime object, parsed by default time parser of ruby with timezone in data' do
+      t = with_timezone("UTC-02") do
+        @d.configure(config_extract_section("time_key" => "t", "time_type" => "string"))
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "2016-09-06 13:27:01 +0900", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 +0900"), t)
+    end
+
+    test 'returns EventTime object, parsed by default time parser of ruby as localtime' do
+      t = with_timezone("UTC-02") do
+        @d.configure(config_extract_section("time_key" => "t", "time_type" => "string"))
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "2016-09-06 13:27:01", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 +0200"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format with timezone' do
+      t = with_timezone("UTC-02") do
+        @d.configure(config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y, %z"))
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016, -0700", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 -0700"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format in localtime without timezone' do
+      t = with_timezone("UTC-02") do
+        @d.configure(config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y"))
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 +0200"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format in utc without timezone, localtime: false' do
+      t = with_timezone("UTC-02") do
+        c = config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y", "localtime" => "false")
+        @d.configure(c)
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 UTC"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format in utc without timezone, utc: true' do
+      t = with_timezone("UTC-02") do
+        c = config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y", "utc" => "true")
+        @d.configure(c)
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 UTC"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format in configured timezone' do
+      t = with_timezone("UTC-02") do
+        c = config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y", "timezone" => "+09:00")
+        @d.configure(c)
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 +0900"), t)
+    end
+  end
+end