fluentd 0.14.6 → 0.14.7

data/test/test_event_router.rb

@@ -307,7 +307,7 @@ class EventRouterTest < ::Test::Unit::TestCase
 
         e = OneEventStream.new(@now, @record)
         assert_rr do
-          mock($log).info("Filtering works with worse performance, because #{[filter_stream].map(&:class)} uses `#filter_stream` method.")
+          mock($log).info("disable filter chain optimization because #{[filter_stream].map(&:class)} uses `#filter_stream` method.")
           mock(filter_stream).filter_stream('test', is_a(OneEventStream)) { e }
           mock(filter).filter_stream('test', is_a(OneEventStream)) { e }
           mock(filter_with_time).filter_stream('test', is_a(OneEventStream)) { e }

data/test/test_formatter.rb

@@ -13,13 +13,6 @@ module FormatterTest
     {'message' => 'awesome', 'greeting' => 'hello'}
   end
 
-  def with_timezone(tz)
-    oldtz, ENV['TZ'] = ENV['TZ'], tz
-    yield
-  ensure
-    ENV['TZ'] = oldtz
-  end
-
   class BaseFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
 

data/test/test_log.rb

@@ -2,9 +2,14 @@ require_relative 'helper'
 require 'fluent/engine'
 require 'fluent/log'
 require 'timecop'
+require 'logger'
 
 class LogTest < Test::Unit::TestCase
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/log/#{ENV['TEST_ENV_NUMBER']}")
+
   def setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
     @log_device = Fluent::Test::DummyLogDevice.new
     @timestamp = Time.parse("2016-04-21 11:58:41 +0900")
     @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
@@ -404,6 +409,84 @@ class LogTest < Test::Unit::TestCase
     # check fluentd log side level is also changed
     assert_equal(Fluent::Log::LEVEL_DEBUG, log.level)
   end
+
+  DAY_SEC = 60 * 60 * 24
+  data(
+    rotate_daily_age: ['daily', 100000, DAY_SEC + 1],
+    rotate_weekly_age: ['weekly', 100000, DAY_SEC * 7 + 1],
+    rotate_monthly_age: ['monthly', 100000, DAY_SEC * 31 + 1],
+    rotate_size: [1, 100, 0, '0'],
+  )
+  def test_log_with_logdevio(expected)
+    with_timezone('utc') do
+      @timestamp = Time.parse("2016-04-21 00:00:00 +0000")
+      @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
+      Timecop.freeze(@timestamp)
+
+      rotate_age, rotate_size, travel_term = expected
+      path = "#{TMP_DIR}/log-dev-io-#{rotate_size}-#{rotate_age}"
+
+      logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
+      logger = ServerEngine::DaemonLogger.new(logdev)
+      log = Fluent::Log.new(logger)
+
+      msg = 'a' * 101
+      log.info msg
+      assert_match msg, File.read(path)
+
+      Timecop.freeze(@timestamp + travel_term)
+
+      msg2 = 'b' * 101
+      log.info msg2
+      c = File.read(path)
+
+      assert_match msg2, c
+      assert_not_equal msg, c
+    end
+  end
+
+  def test_log_rotates_specifed_size_with_logdevio
+    with_timezone('utc') do
+      rotate_age = 2
+      rotate_size = 100
+      path = "#{TMP_DIR}/log-dev-io-#{rotate_size}-#{rotate_age}"
+      path0 = path + '.0'
+      path1 = path + '.1'
+
+      logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
+      logger = ServerEngine::DaemonLogger.new(logdev)
+      log = Fluent::Log.new(logger)
+
+      msg = 'a' * 101
+      log.info msg
+      assert_match msg, File.read(path)
+      assert_true File.exist?(path)
+      assert_true !File.exist?(path0)
+      assert_true !File.exist?(path1)
+
+      # create log.0
+      msg2 = 'b' * 101
+      log.info msg2
+      c = File.read(path)
+      c0 = File.read(path0)
+      assert_match msg2, c
+      assert_match msg, c0
+      assert_true File.exist?(path)
+      assert_true File.exist?(path0)
+      assert_true !File.exist?(path1)
+
+      # rotate
+      msg3 = 'c' * 101
+      log.info msg3
+      c = File.read(path)
+      c0 = File.read(path0)
+      assert_match msg3, c
+      assert_match msg2, c0
+      assert_true File.exist?(path)
+      assert_true File.exist?(path0)
+      assert_true !File.exist?(path1)
+    end
+  end
 end
 
 class PluginLoggerTest < Test::Unit::TestCase
@@ -570,3 +653,41 @@ class PluginLoggerMixinTest < Test::Unit::TestCase
     plugin.terminate
   end
 end
+
+class LogDeviceIOTest < Test::Unit::TestCase
+  test 'flush' do
+    io = StringIO.new
+    logdev = Fluent::LogDeviceIO.new(io)
+    assert_equal io, logdev.flush
+
+    io.instance_eval { undef :flush }
+    logdev = Fluent::LogDeviceIO.new(io)
+    assert_raise NoMethodError do
+      logdev.flush
+    end
+  end
+
+  test 'tty?' do
+    io = StringIO.new
+    logdev = Fluent::LogDeviceIO.new(io)
+    assert_equal io.tty?, logdev.tty?
+
+    io.instance_eval { undef :tty? }
+    logdev = Fluent::LogDeviceIO.new(io)
+    assert_raise NoMethodError do
+      logdev.tty?
+    end
+  end
+
+  test 'sync=' do
+    io = StringIO.new
+    logdev = Fluent::LogDeviceIO.new(io)
+    assert_true logdev.sync = true
+
+    io.instance_eval { undef :sync= }
+    logdev = Fluent::LogDeviceIO.new(io)
+    assert_raise NoMethodError do
+      logdev.sync = true
+    end
+  end
+end

data/test/test_plugin_classes.rb

@@ -1,6 +1,7 @@
 require_relative 'helper'
 require 'fluent/plugin/input'
 require 'fluent/plugin/output'
+require 'fluent/plugin/bare_output'
 require 'fluent/plugin/filter'
 
 module FluentTest
@@ -48,6 +49,67 @@ module FluentTest
     end
   end
 
+  class FluentTestDynamicOutput < ::Fluent::Plugin::BareOutput
+    ::Fluent::Plugin.register_output('test_dynamic_out', self)
+
+    attr_reader :child
+    attr_reader :started
+
+    def start
+      super
+      @started = true
+      @child = Fluent::Plugin.new_output('copy')
+      conf = config_element('DYNAMIC', '', {}, [
+          config_element('store', '', {'@type' => 'test_out', '@id' => 'dyn_out1'}),
+          config_element('store', '', {'@type' => 'test_out', '@id' => 'dyn_out2'}),
+      ])
+      @child.configure(conf)
+      @child.start
+    end
+
+    def after_start
+      super
+      @child.after_start
+    end
+
+    def stop
+      super
+      @child.stop
+    end
+
+    def before_shutdown
+      super
+      @child.before_shutdown
+    end
+
+    def shutdown
+      @started = false
+      super
+      @child.shutdown
+    end
+
+    def after_shutdown
+      super
+      @child.after_shutdown
+    end
+
+    def close
+      super
+      @child.close
+    end
+
+    def terminate
+      super
+      @child.terminate
+    end
+
+    def process(tag, es)
+      es.each do |time, record|
+        @events[tag] << record
+      end
+    end
+  end
+
   class FluentTestErrorOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_error', self)
 

data/test/test_root_agent.rb

@@ -89,6 +89,41 @@ EOC
       end
     end
 
+    test 'raises configuration error if there are two same label section' do
+      conf = <<-EOC
+<source>
+  @type test_in
+  @label @test
+</source>
+<label @test>
+  @type test_out
+</label>
+<label @test>
+  @type test_out
+</label>
+EOC
+      errmsg = "Section <label @test> appears twice"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error if there are not match sections in label section' do
+      conf = <<-EOC
+<source>
+  @type test_in
+  @label @test
+</source>
+<label @test>
+  @type test_out
+</label>
+EOC
+      errmsg = "Missing <match> sections in <label @test> section"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
     test 'with plugins' do
       # check @type and type in one configuration
       conf = <<-EOC
@@ -316,5 +351,95 @@ EOC
       assert_equal [true], label_filters.map{|i| i.terminated? }
       assert_equal [true, true, true], label_outputs.map{|i| i.terminated? }
     end
+
+    test 'plugin #shutdown is not called twice' do
+      assert_equal 1, @ra.inputs.size
+      assert_equal 0, @ra.filters.size
+      assert_equal 0, @ra.outputs.size
+      assert_equal 1, @ra.labels.size
+      assert_equal '@testing', @ra.labels.keys.first
+      assert_equal 1, @ra.labels.values.first.filters.size
+      assert_equal 3, @ra.labels.values.first.outputs.size
+
+      @ra.start
+
+      old_level = @ra.log.level
+      begin
+        @ra.log.instance_variable_get(:@logger).level = Fluent::Log::LEVEL_INFO - 1
+        assert_equal Fluent::Log::LEVEL_INFO, @ra.log.level
+
+        @ra.log.out.flush_logs = false
+
+        @ra.shutdown
+
+        test_out1_shutdown_logs = @ra.log.out.logs.select{|line| line =~ /shutting down output plugin type=:test_out plugin_id="test_out1"/ }
+        assert_equal 1, test_out1_shutdown_logs.size
+      ensure
+        @ra.log.out.flush_logs = true
+        @ra.log.out.reset
+        @ra.log.level = old_level
+      end
+    end
+  end
+
+  sub_test_case 'configured with MultiOutput plugin which creates plugin instances dynamically' do
+    setup do
+      @ra = RootAgent.new(log: $log)
+      stub(Engine).root_agent { @ra }
+      @ra.configure(Config.parse(<<-EOC, "(test)", "(test_dir)", true))
+<source>
+  @type test_in
+  @id test_in
+  @label @testing
+</source>
+<label @testing>
+  <match **>
+    @type test_dynamic_out
+    @id test_dyn
+  </match>
+</label>
+EOC
+      @ra
+    end
+
+    test 'plugin status with multi output' do
+      assert_equal 1, @ra.inputs.size
+      assert_equal 0, @ra.filters.size
+      assert_equal 0, @ra.outputs.size
+      assert_equal 1, @ra.labels.size
+      assert_equal '@testing', @ra.labels.keys.first
+      assert_equal 0, @ra.labels.values.first.filters.size
+      assert_equal 1, @ra.labels.values.first.outputs.size
+
+      dyn_out = @ra.labels.values.first.outputs.first
+      assert_nil dyn_out.child
+
+      @ra.start
+
+      assert_equal 1, @ra.labels.values.first.outputs.size
+
+      assert dyn_out.child
+      assert_false dyn_out.child.outputs_statically_created
+      assert_equal 2, dyn_out.child.outputs.size
+
+      assert_equal true, dyn_out.child.outputs[0].started?
+      assert_equal true, dyn_out.child.outputs[1].started?
+      assert_equal true, dyn_out.child.outputs[0].after_started?
+      assert_equal true, dyn_out.child.outputs[1].after_started?
+
+      @ra.shutdown
+
+      assert_equal 1, @ra.labels.values.first.outputs.size
+
+      assert_false dyn_out.child.outputs_statically_created
+      assert_equal 2, dyn_out.child.outputs.size
+
+      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.stopped? }
+      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.before_shutdown? }
+      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.shutdown? }
+      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.after_shutdown? }
+      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.closed? }
+      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.terminated? }
+    end
   end
 end

data/test/test_supervisor.rb

@@ -14,10 +14,11 @@ class SupervisorTest < ::Test::Unit::TestCase
   include ServerModule
   include WorkerModule
 
-  TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/supervisor#{ENV['TEST_ENV_NUMBER']}")
 
   def setup
-    FileUtils.mkdir_p('test/tmp/supervisor')
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
   end
 
   def write_config(path, data)
@@ -319,6 +320,28 @@ class SupervisorTest < ::Test::Unit::TestCase
     # test that DamonLogger#level= overwrites Fluent.log#level
     logger.level = 'debug'
     assert_equal Fluent::Log::LEVEL_DEBUG, $log.level
+
+    assert_equal 5, logger.instance_variable_get(:@rotate_age)
+    assert_equal 1048576, logger.instance_variable_get(:@rotate_size)
+  end
+
+  data(
+    daily_age: 'daily',
+    weekly_age: 'weekly',
+    monthly_age: 'monthly',
+    integer_age: 2,
+  )
+  def test_logger_with_rotate_age_and_rotate_size(rotate_age)
+    opts = Fluent::Supervisor.default_options.merge(
+      log_path: "#{TMP_DIR}/test", log_rotate_age: rotate_age, log_rotate_size: 10
+    )
+    sv = Fluent::Supervisor.new(opts)
+    log = sv.instance_variable_get(:@log)
+    log.init
+
+    assert_equal Fluent::LogDeviceIO, $log.out.class
+    assert_equal rotate_age, $log.out.instance_variable_get(:@shift_age)
+    assert_equal 10, $log.out.instance_variable_get(:@shift_size)
   end
 
   def create_debug_dummy_logger

data/test/test_time_formatter.rb

@@ -3,13 +3,6 @@ require 'fluent/test'
 require 'fluent/time'
 
 class TimeFormatterTest < ::Test::Unit::TestCase
-  def with_timezone(tz)
-    oldtz, ENV['TZ'] = ENV['TZ'], tz
-    yield
-  ensure
-    ENV['TZ'] = oldtz
-  end
-
   def setup
     @time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i
     @fmt  = "%Y%m%d %H%M%z"  # YYYYMMDD HHMM[+-]HHMM
@@ -183,4 +176,107 @@ class TimeFormatterTest < ::Test::Unit::TestCase
     formatter = Fluent::TimeFormatter.new("%Y%m%d %H%M.%N", false, nil)
     assert_equal("20140927 0000.000000000", formatter.format(time))
   end
+
+  sub_test_case 'TimeMixin::Formatter' do
+    class DummyForTimeFormatter
+      include Fluent::Configurable
+      include Fluent::TimeMixin::Formatter
+    end
+
+    test 'provides configuration parameters for TimeFormatter with default values for localtime' do
+      str = with_timezone("UTC+07") do
+        i = DummyForTimeFormatter.new
+        i.configure(config_element('format'))
+
+        assert_nil   i.time_format
+        assert_true  i.localtime
+        assert_false i.utc
+        assert_nil   i.timezone
+
+        fmt = i.time_formatter_create
+        fmt.format(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+      end
+      assert_equal "2016-09-02T11:42:31-07:00", str
+    end
+
+    test 'provides configuration parameters for TimeFormatter, configurable for any time format' do
+      str = with_timezone("UTC+07") do
+        i = DummyForTimeFormatter.new
+        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z'}))
+
+        fmt = i.time_formatter_create
+        fmt.format(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+      end
+      assert_equal "2016-09-02 11:42:31.012345678 -0700", str
+    end
+
+    test 'provides configuration parameters for TimeFormatter, configurable for UTC' do
+      str = with_timezone("UTC+07") do
+        i = DummyForTimeFormatter.new
+        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'utc' => 'true'}))
+
+        fmt = i.time_formatter_create
+        fmt.format(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+      end
+      assert_equal "2016-09-02 18:42:31.012345678 +0000", str
+    end
+
+    test 'provides configuration parameters for TimeFormatter, configurable for any timezone' do
+      str = with_timezone("UTC+07") do
+        i = DummyForTimeFormatter.new
+        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'timezone' => '+0900'}))
+
+        fmt = i.time_formatter_create
+        fmt.format(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+      end
+      assert_equal "2016-09-03 03:42:31.012345678 +0900", str
+    end
+
+    test '#time_formatter_create returns TimeFormatter with specified time format and timezone' do
+      str = with_timezone("UTC+07") do
+        i = DummyForTimeFormatter.new
+        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'timezone' => '+0900'}))
+
+        fmt = i.time_formatter_create(format: '%m/%d/%Y %H-%M-%S %N', timezone: '+0000')
+        fmt.format(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+      end
+      assert_equal "09/02/2016 18-42-31 012345678", str
+    end
+
+    test '#time_formatter_create returns TimeFormatter with localtime besides any configuration parameters' do
+      str = with_timezone("UTC+07") do
+        i = DummyForTimeFormatter.new
+        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'utc' => 'true'}))
+
+        fmt = i.time_formatter_create(format: '%m/%d/%Y %H-%M-%S %N', force_localtime: true)
+        fmt.format(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+      end
+      assert_equal "09/02/2016 11-42-31 012345678", str
+
+      str = with_timezone("UTC+07") do
+        i = DummyForTimeFormatter.new
+        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'timezone' => '+0900'}))
+
+        fmt = i.time_formatter_create(format: '%m/%d/%Y %H-%M-%S %N', force_localtime: true)
+        fmt.format(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+      end
+      assert_equal "09/02/2016 11-42-31 012345678", str
+    end
+  end
+
+  test '#time_formatter_create returns NumericTimeFormatter to format time as unixtime when time_type unixtime specified' do
+    i = DummyForTimeFormatter.new
+    i.configure(config_element('format', '', {'time_type' => 'unixtime'}))
+    fmt = i.time_formatter_create
+    time = event_time("2016-10-03 20:08:30.123456789 +0100", format: '%Y-%m-%d %H:%M:%S.%N %z')
+    assert_equal "#{time.sec}", fmt.format(time)
+  end
+
+  test '#time_formatter_create returns NumericTimeFormatter to format time as float when time_type float specified' do
+    i = DummyForTimeFormatter.new
+    i.configure(config_element('format', '', {'time_type' => 'float'}))
+    fmt = i.time_formatter_create
+    time = event_time("2016-10-03 20:08:30.123456789 +0100", format: '%Y-%m-%d %H:%M:%S.%N %z')
+    assert_equal "#{time.sec}.#{time.nsec}", fmt.format(time)
+  end
 end