fluentd 0.14.1 → 0.14.2

data/test/plugin_helper/test_compat_parameters.rb

@@ -2,6 +2,8 @@ require_relative '../helper'
 require 'fluent/plugin_helper/compat_parameters'
 require 'fluent/plugin/base'
 
+require 'time'
+
 class CompatParameterTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
@@ -19,46 +21,53 @@ class CompatParameterTest < Test::Unit::TestCase
     end
   end
 
-  class Dummy0 < Fluent::Plugin::Output
+  class DummyO0 < Fluent::Plugin::Output
     helpers :compat_parameters
-    def compat_parameters_default_chunk_key
-      ''
-    end
-    def write(chunk)
-      # dummy
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: '')
+      super
     end
+    def write(chunk); end # dummy
   end
-  class Dummy1 < Fluent::Plugin::Output
+  class DummyO1 < Fluent::Plugin::Output
     helpers :compat_parameters
-    def compat_parameters_default_chunk_key
-      'time'
-    end
-    def write(chunk)
-      # dummy
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'time')
+      super
     end
+    def write(chunk); end # dummy
   end
-  class Dummy2 < Fluent::Plugin::Output
+  class DummyO2 < Fluent::Plugin::Output
     helpers :compat_parameters
-    # for test to assume default key time by 'time_slice_format'
-    def write(chunk)
-      # dummy
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'time')
+      super
     end
+    def write(chunk); end # dummy
   end
-  class Dummy3 < Fluent::Plugin::Output
+  class DummyO3 < Fluent::Plugin::Output
     helpers :compat_parameters
-    def compat_parameters_default_chunk_key
-      'tag'
-    end
-    def write(chunk)
-      # dummy
+    def configure(conf)
+      compat_parameters_buffer(conf, default_chunk_key: 'tag')
+      super
     end
+    def write(chunk); end # dummy
+  end
+  class DummyO4 < Fluent::Plugin::Output
+    helpers :compat_parameters, :inject, :formatter
+    attr_reader :f
+    def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject, :formatter, default_chunk_key: 'tag')
+      super
+    end
+    def start
+      super
+      @f = formatter_create()
+    end
+    def write(chunk); end # dummy
   end
 
-  sub_test_case 'plugins which does not have default chunk key' do
-    setup do
-      @p = Dummy0
-    end
-
+  sub_test_case 'output plugins which does not have default chunk key' do
     test 'plugin helper converts parameters into plugin configuration parameters' do
       hash = {
         'num_threads' => 8,
@@ -68,7 +77,7 @@ class CompatParameterTest < Test::Unit::TestCase
         'flush_at_shutdown' => 'yes',
       }
       conf = config_element('ROOT', '', hash)
-      @i = @p.new
+      @i = DummyO0.new
       @i.configure(conf)
 
       assert_equal 'memory', @i.buffer_config[:@type]
@@ -82,11 +91,7 @@ class CompatParameterTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'plugins which has default chunk key: time' do
-    setup do
-      @p = Dummy1
-    end
-
+  sub_test_case 'output plugins which has default chunk key: time' do
     test 'plugin helper converts parameters into plugin configuration parameters' do
       hash = {
         'buffer_type' => 'file',
@@ -96,7 +101,7 @@ class CompatParameterTest < Test::Unit::TestCase
         'buffer_queue_full_action' => 'block',
       }
       conf = config_element('ROOT', '', hash)
-      @i = @p.new
+      @i = DummyO1.new
       @i.configure(conf)
 
       assert_equal 'file', @i.buffer_config[:@type]
@@ -112,11 +117,7 @@ class CompatParameterTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'plugins which does not have default chunk key' do
-    setup do
-      @p = Dummy2
-    end
-
+  sub_test_case 'output plugins which does not have default chunk key' do
     test 'plugin helper converts parameters into plugin configuration parameters' do
       hash = {
         'buffer_type' => 'file',
@@ -127,7 +128,7 @@ class CompatParameterTest < Test::Unit::TestCase
         'buffer_queue_full_action' => 'drop_oldest_chunk',
       }
       conf = config_element('ROOT', '', hash)
-      @i = @p.new
+      @i = DummyO2.new
       @i.configure(conf)
 
       assert_equal 'file', @i.buffer_config[:@type]
@@ -144,11 +145,7 @@ class CompatParameterTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'plugins which has default chunk key: tag' do
-    setup do
-      @p = Dummy3
-    end
-
+  sub_test_case 'output plugins which has default chunk key: tag' do
     test 'plugin helper converts parameters into plugin configuration parameters' do
       hash = {
         'buffer_type' => 'memory',
@@ -158,7 +155,7 @@ class CompatParameterTest < Test::Unit::TestCase
         'queued_chunk_flush_interval' => '0.5',
       }
       conf = config_element('ROOT', '', hash)
-      @i = @p.new
+      @i = DummyO3.new
       @i.configure(conf)
 
       assert_equal 'memory', @i.buffer_config[:@type]
@@ -172,4 +169,72 @@ class CompatParameterTest < Test::Unit::TestCase
       assert_equal [], @i.chunk_keys
     end
   end
+
+  sub_test_case 'output plugins which has default chunk key: tag, and enables inject and formatter' do
+    test 'plugin helper converts parameters into plugin configuration parameters for all of buffer, inject and formatter' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => File.expand_path('../../tmp/compat_parameters/mybuffer.*.log', __FILE__),
+        'num_threads' => '10',
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'include_time_key' => 'true', # default time_key 'time' and default time format (iso8601: 2016-06-24T15:57:38) at localtime
+        'include_tag_key' => 'yes', # default tag_key 'tag'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO4.new
+      @i.configure(conf)
+      @i.start
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      formatter = @i.f
+      assert{ formatter.is_a? Fluent::Plugin::LabeledTSVFormatter }
+      assert_equal ',', @i.f.delimiter
+      assert_equal '%', @i.f.label_delimiter
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      t = event_time('2016-06-24 16:05:01') # localtime
+      iso8601str = Time.at(t.to_i).iso8601
+      formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+    end
+
+    test 'plugin helper setups time injecting as unix time (integer from epoch)' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => File.expand_path('../../tmp/compat_parameters/mybuffer.*.log', __FILE__),
+        'num_threads' => '10',
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'include_time_key' => 'true', # default time_key 'time' and default time format (iso8601: 2016-06-24T15:57:38) at localtime
+        'include_tag_key' => 'yes', # default tag_key 'tag'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyO4.new
+      @i.configure(conf)
+      @i.start
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      formatter = @i.f
+      assert{ formatter.is_a? Fluent::Plugin::LabeledTSVFormatter }
+      assert_equal ',', @i.f.delimiter
+      assert_equal '%', @i.f.label_delimiter
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      t = event_time('2016-06-24 16:05:01') # localtime
+      iso8601str = Time.at(t.to_i).iso8601
+      formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+    end
+  end
 end

data/test/plugin_helper/test_formatter.rb

@@ -19,6 +19,13 @@ class FormatterHelperTest < Test::Unit::TestCase
     helpers :formatter
   end
 
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :formatter
+    config_section :format do
+      config_set_default :@type, 'example2'
+    end
+  end
+
   setup do
     @d = nil
   end
@@ -37,6 +44,39 @@ class FormatterHelperTest < Test::Unit::TestCase
     assert_equal 0, d._formatters.size
   end
 
+  test 'can override default configuration parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    assert_equal [], d.formatter_configs
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('format', '', {}, [])]))
+    assert_raise NoMethodError do
+      d.format
+    end
+    assert_equal 1, d.formatter_configs.size
+    assert_equal 'example2', d.formatter_configs.first[:@type]
+  end
+
+  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.formatter_create(conf: config_element('format', '', {'@type' => 'example'}), default_type: 'example2')
+    assert{ i.is_a?(ExampleFormatter) }
+
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.formatter_create(conf: nil, default_type: 'example2')
+    assert{ i.is_a?(Example2Formatter) }
+  end
+
+  test 'raises config error if config section is specified, but @type is not specified' do
+    d = Dummy2.new
+    d.configure(config_element())
+    assert_raise Fluent::ConfigError.new("@type is required in <format>") do
+      d.formatter_create(conf: config_element('format', '', {}), default_type: 'example2')
+    end
+  end
+
   test 'can be configured without format sections' do
     d = Dummy.new
     assert_nothing_raised do

data/test/plugin_helper/test_inject.rb

@@ -8,6 +8,13 @@ class InjectHelperTest < Test::Unit::TestCase
     helpers :inject
   end
 
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :inject
+    config_section :inject do
+      config_set_default :hostname_key, 'host'
+    end
+  end
+
   def config_inject_section(hash = {})
     config_element('ROOT', '', {}, [config_element('inject', '', hash)])
   end
@@ -26,6 +33,17 @@ class InjectHelperTest < Test::Unit::TestCase
     end
   end
 
+  test 'can override default parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    d.configure(config_element())
+    assert_nil d.inject_config
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('inject')]))
+    assert d.inject_config
+    assert_equal 'host', d.inject_config.hostname_key
+  end
+
   test 'do nothing in default' do
     @d.configure(config_inject_section())
     @d.start
@@ -75,7 +93,7 @@ class InjectHelperTest < Test::Unit::TestCase
     assert_not_nil @d.instance_eval{ @_inject_time_formatter }
   end
 
-  sub_test_case 'using inject_record' do
+  sub_test_case 'using inject_values_to_record' do
     test 'injects hostname automatically detected' do
       detected_hostname = `hostname`.chomp
       @d.configure(config_inject_section("hostname_key" => "host"))
@@ -204,7 +222,8 @@ class InjectHelperTest < Test::Unit::TestCase
       assert_equal record.merge(injected), @d.inject_values_to_record('tag', time, record)
     end
   end
-  sub_test_case 'using inject_event_stream' do
+
+  sub_test_case 'using inject_values_to_event_stream' do
     local_timezone = Time.now.strftime('%z')
     time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
     time_subsecond = 320_101_224
@@ -385,4 +404,84 @@ class InjectHelperTest < Test::Unit::TestCase
       assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
     end
   end
+
+  sub_test_case 'time formatting with modified timezone' do
+    setup do
+      @time = event_time("2014-09-27 00:00:00 +00:00").to_i
+    end
+
+    def with_timezone(tz)
+      oldtz, ENV['TZ'] = ENV['TZ'], tz
+      yield
+    ensure
+      ENV['TZ'] = oldtz
+    end
+
+    def format(conf)
+      @d.configure(config_inject_section(
+          "hostname_key" => "hostnamedata",
+          "hostname" => "myname.local",
+          "tag_key" => "tagdata",
+          "time_key" => "timedata",
+          "time_type" => "string",
+          "time_format" => "%Y_%m_%d %H:%M:%S.%N %z",
+          "timezone" => "+0000",
+      ))
+      @d.start
+
+      record = {"key1" => "value1", "key2" => 2}
+      injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => "2016_06_20 23:10:11.320101224 +0000"}
+      assert_equal record.merge(injected), @d.inject_values_to_record('tag', time, record)
+
+
+      d = create_driver({'include_time_key' => true}.merge(conf))
+      formatted = d.instance.format("tag", @time, {})
+      # Drop the leading "time:" and the trailing "\n".
+      formatted[5..-2]
+    end
+
+    def test_nothing_specified_about_time_formatting
+      with_timezone("UTC-01") do
+        # 'localtime' is true by default.
+        @d.configure(config_inject_section("time_key" => "t", "time_type" => "string"))
+        @d.start
+        record = @d.inject_values_to_record('tag', @time, {"message" => "yay"})
+
+        assert_equal("2014-09-27T01:00:00+01:00", record['t'])
+      end
+    end
+
+    def test_utc
+      with_timezone("UTC-01") do
+        # 'utc' takes precedence over 'localtime'.
+        @d.configure(config_inject_section("time_key" => "t", "time_type" => "string", "utc" => "true"))
+        @d.start
+        record = @d.inject_values_to_record('tag', @time, {"message" => "yay"})
+
+        assert_equal("2014-09-27T00:00:00Z", record['t'])
+      end
+    end
+
+    def test_timezone
+      with_timezone("UTC-01") do
+        # 'timezone' takes precedence over 'localtime'.
+        @d.configure(config_inject_section("time_key" => "t", "time_type" => "string", "timezone" => "+02"))
+        @d.start
+        record = @d.inject_values_to_record('tag', @time, {"message" => "yay"})
+
+        assert_equal("2014-09-27T02:00:00+02:00", record['t'])
+      end
+    end
+
+    def test_utc_timezone
+      with_timezone("UTC-01") do
+        # 'timezone' takes precedence over 'utc'.
+        @d.configure(config_inject_section("time_key" => "t", "time_type" => "string", "timezone" => "Asia/Tokyo", "utc" => "true"))
+        @d.start
+        record = @d.inject_values_to_record('tag', @time, {"message" => "yay"})
+
+        assert_equal("2014-09-27T09:00:00+09:00", record['t'])
+      end
+    end
+  end
 end

data/test/plugin_helper/test_parser.rb

@@ -30,6 +30,13 @@ class ParserHelperTest < Test::Unit::TestCase
     helpers :parser
   end
 
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :parser
+    config_section :parse do
+      config_set_default :@type, 'example2'
+    end
+  end
+
   setup do
     @d = nil
   end
@@ -48,6 +55,39 @@ class ParserHelperTest < Test::Unit::TestCase
     assert_equal 0, d._parsers.size
   end
 
+  test 'can override default configuration parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    assert_equal [], d.parser_configs
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('parse', '', {}, [])]))
+    assert_raise NoMethodError do
+      d.parse
+    end
+    assert_equal 1, d.parser_configs.size
+    assert_equal 'example2', d.parser_configs.first[:@type]
+  end
+
+  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.parser_create(conf: config_element('parse', '', {'@type' => 'example'}), default_type: 'example2')
+    assert{ i.is_a?(ExampleParser) }
+
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.parser_create(conf: nil, default_type: 'example2')
+    assert{ i.is_a?(Example2Parser) }
+  end
+
+  test 'raises config error if config section is specified, but @type is not specified' do
+    d = Dummy2.new
+    d.configure(config_element())
+    assert_raise Fluent::ConfigError.new("@type is required in <parse>") do
+      d.parser_create(conf: config_element('parse', '', {}), default_type: 'example2')
+    end
+  end
+
   test 'can be configured without parse sections' do
     d = Dummy.new
     assert_nothing_raised do