fluentd 0.12.40 → 0.14.0

data/test/plugin_helper/test_compat_parameters.rb

@@ -0,0 +1,175 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/compat_parameters'
+require 'fluent/plugin/base'
+
+class CompatParameterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @i = nil
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+  end
+
+  class Dummy0 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def compat_parameters_default_chunk_key
+      ''
+    end
+    def write(chunk)
+      # dummy
+    end
+  end
+  class Dummy1 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def compat_parameters_default_chunk_key
+      'time'
+    end
+    def write(chunk)
+      # dummy
+    end
+  end
+  class Dummy2 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    # for test to assume default key time by 'time_slice_format'
+    def write(chunk)
+      # dummy
+    end
+  end
+  class Dummy3 < Fluent::Plugin::Output
+    helpers :compat_parameters
+    def compat_parameters_default_chunk_key
+      'tag'
+    end
+    def write(chunk)
+      # dummy
+    end
+  end
+
+  sub_test_case 'plugins which does not have default chunk key' do
+    setup do
+      @p = Dummy0
+    end
+
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'num_threads' => 8,
+        'flush_interval' => '10s',
+        'buffer_chunk_limit' => '8m',
+        'buffer_queue_limit' => '1024',
+        'flush_at_shutdown' => 'yes',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = @p.new
+      @i.configure(conf)
+
+      assert_equal 'memory', @i.buffer_config[:@type]
+      assert_equal [], @i.buffer_config.chunk_keys
+      assert_equal 8, @i.buffer_config.flush_thread_count
+      assert_equal 10, @i.buffer_config.flush_interval
+      assert @i.buffer_config.flush_at_shutdown
+
+      assert_equal 8*1024*1024, @i.buffer.chunk_limit_size
+      assert_equal 1024, @i.buffer.queue_length_limit
+    end
+  end
+
+  sub_test_case 'plugins which has default chunk key: time' do
+    setup do
+      @p = Dummy1
+    end
+
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => '/tmp/mybuffer',
+        'disable_retry_limit' => 'yes',
+        'max_retry_wait' => '1h',
+        'buffer_queue_full_action' => 'block',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = @p.new
+      @i.configure(conf)
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 24*60*60, @i.buffer_config.timekey
+      assert @i.buffer_config.retry_forever
+      assert_equal 60*60, @i.buffer_config.retry_max_interval
+      assert_equal :block, @i.buffer_config.overflow_action
+
+      assert !@i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      assert_equal '/tmp/mybuffer/buffer.*.log', @i.buffer.path
+    end
+  end
+
+  sub_test_case 'plugins which does not have default chunk key' do
+    setup do
+      @p = Dummy2
+    end
+
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'file',
+        'buffer_path' => '/tmp/mybuffer',
+        'time_slice_format' => '%Y%m%d%H',
+        'time_slice_wait' => '10',
+        'retry_limit' => '1024',
+        'buffer_queue_full_action' => 'drop_oldest_chunk',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = @p.new
+      @i.configure(conf)
+
+      assert_equal 'file', @i.buffer_config[:@type]
+      assert_equal 60*60, @i.buffer_config.timekey
+      assert_equal 10, @i.buffer_config.timekey_wait
+      assert_equal 1024, @i.buffer_config.retry_max_times
+      assert_equal :drop_oldest_chunk, @i.buffer_config.overflow_action
+
+      assert @i.chunk_key_time
+      assert !@i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+
+      assert_equal '/tmp/mybuffer/buffer.*.log', @i.buffer.path
+    end
+  end
+
+  sub_test_case 'plugins which has default chunk key: tag' do
+    setup do
+      @p = Dummy3
+    end
+
+    test 'plugin helper converts parameters into plugin configuration parameters' do
+      hash = {
+        'buffer_type' => 'memory',
+        'num_threads' => '10',
+        'flush_interval' => '10s',
+        'try_flush_interval' => '0.1',
+        'queued_chunk_flush_interval' => '0.5',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = @p.new
+      @i.configure(conf)
+
+      assert_equal 'memory', @i.buffer_config[:@type]
+      assert_equal 10, @i.buffer_config.flush_thread_count
+      assert_equal 10, @i.buffer_config.flush_interval
+      assert_equal 0.1, @i.buffer_config.flush_thread_interval
+      assert_equal 0.5, @i.buffer_config.flush_thread_burst_interval
+
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+    end
+  end
+end

data/test/plugin_helper/test_event_emitter.rb

@@ -0,0 +1,51 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/event_emitter'
+require 'fluent/plugin/base'
+
+class EventEmitterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  class Dummy0 < Fluent::Plugin::TestBase
+  end
+
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :event_emitter
+  end
+
+  test 'can be instantiated to be able to emit with router' do
+    d0 = Dummy0.new
+    assert d0.respond_to?(:has_router?)
+    assert !d0.has_router?
+    assert !d0.respond_to?(:router)
+
+    d1 = Dummy.new
+    assert d1.respond_to?(:has_router?)
+    assert d1.has_router?
+    assert d1.respond_to?(:router)
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can be configured with valid router' do
+    d1 = Dummy.new
+    assert d1.has_router?
+    assert_nil d1.router
+
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+
+    assert d1.router
+
+    d1.shutdown
+
+    assert d1.router
+
+    d1.close
+
+    assert_nil d1.router
+
+    d1.terminate
+  end
+end

data/test/plugin_helper/test_event_loop.rb

@@ -0,0 +1,52 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/event_loop'
+require 'fluent/plugin/base'
+
+class EventLoopTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :event_loop
+    def configure(conf)
+      super
+      @_event_loop_run_timeout = 0.1
+    end
+  end
+
+  test 'can be instantiated to be able to create event loop' do
+    d1 = Dummy.new
+    assert d1.respond_to?(:event_loop_attach)
+    assert d1.respond_to?(:event_loop_running?)
+    assert d1.respond_to?(:_event_loop)
+    assert d1._event_loop
+    assert !d1.event_loop_running?
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'can run event loop by start, stop by shutdown/close and clear by terminate' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.event_loop_running?
+
+    d1.start
+    d1.event_loop_wait_until_start
+
+    assert d1.event_loop_running?
+    assert_equal 1, d1._event_loop.watchers.size
+
+    d1.shutdown
+    d1.close
+
+    assert !d1.event_loop_running?
+
+    d1.terminate
+
+    assert_nil d1._event_loop
+  end
+end

data/test/plugin_helper/test_retry_state.rb

@@ -0,0 +1,399 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/retry_state'
+require 'fluent/plugin/base'
+
+require 'time'
+
+class Fluent::PluginHelper::RetryState::RetryStateMachine
+  def override_current_time(time)
+    (class << self; self; end).module_eval do
+      alias_method(:current_time_orig, :current_time)
+      define_method(:current_time){ time }
+    end
+  end
+end
+
+class RetryStateHelperTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :retry_state
+  end
+
+  setup do
+    @d = Dummy.new
+  end
+
+  test 'randomize can generate value within specified +/- range' do
+    s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30) # default enabled w/ 0.125
+    500.times do
+      r = s.randomize(1000)
+      assert{ r >= 875 && r < 1125 }
+    end
+
+    s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30, randomize_width: 0.25)
+    500.times do
+      r = s.randomize(1000)
+      assert{ r >= 750 && r < 1250 }
+    end
+  end
+
+  test 'plugin can create retry_state machine' do
+    s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30)
+    # attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_times
+
+    assert_equal :t1, s.title
+    start_time = s.start
+
+    assert_equal 0, s.steps
+    assert_equal (start_time + 0.1).to_i, s.next_time.to_i
+    assert_equal (start_time + 0.1).nsec, s.next_time.nsec
+    assert_equal (start_time + 30), s.timeout_at
+
+    assert_equal :primary, s.current
+    assert{ s.is_a? Fluent::PluginHelper::RetryState::ExponentialBackOffRetry }
+  end
+
+  test 'periodic retries' do
+    s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false)
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+    assert_equal (dummy_current_time + 29), s.timeout_at
+    assert_equal (dummy_current_time + 3), s.next_time
+
+    i = 1
+    while i < 9
+      s.override_current_time(s.next_time)
+      s.step
+      assert_equal i, s.steps
+      assert_equal (s.current_time + 3), s.next_time
+      assert !s.limit?
+      i += 1
+    end
+
+    assert_equal 9, i
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal s.timeout_at, s.next_time
+    assert s.limit?
+  end
+
+  test 'periodic retries with max_steps' do
+    s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false, max_steps: 5)
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+    assert_equal (dummy_current_time + 29), s.timeout_at
+    assert_equal (dummy_current_time + 3), s.next_time
+
+    i = 1
+    while i < 5
+      s.override_current_time(s.next_time)
+      s.step
+      assert_equal i, s.steps
+      assert_equal (s.current_time + 3), s.next_time
+      assert !s.limit?
+      i += 1
+    end
+
+    assert_equal 5, i
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal (s.current_time + 3), s.next_time
+    assert s.limit?
+  end
+
+  test 'periodic retries with secondary' do
+    s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true) # threshold 0.8
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+    assert_equal (dummy_current_time + 100), s.timeout_at
+    assert_equal (dummy_current_time + 100 * 0.8), s.secondary_transition_at
+
+    assert_equal (dummy_current_time + 3), s.next_time
+    assert !s.secondary?
+
+    i = 1
+    while i < 26
+      s.override_current_time(s.next_time)
+      assert !s.secondary?
+
+      s.step
+      assert_equal i, s.steps
+      assert_equal (s.current_time + 3), s.next_time
+      assert !s.limit?
+      i += 1
+    end
+
+    assert_equal 26, i
+    s.override_current_time(s.next_time) # 78
+    assert !s.secondary?
+
+    s.step
+    assert_equal 26, s.steps
+    assert_equal s.secondary_transition_at, s.next_time
+    assert !s.limit?
+
+    i += 1
+    assert_equal 27, i
+    s.override_current_time(s.next_time) # 80
+    assert s.secondary?
+
+    s.step
+    assert_equal (s.current_time + 3), s.next_time
+    assert_equal s.steps, s.secondary_transition_steps
+    assert !s.limit?
+
+    i += 1
+
+    while i < 33
+      s.override_current_time(s.next_time)
+      assert s.secondary?
+
+      s.step
+      assert_equal (s.current_time + 3), s.next_time
+      assert !s.limit?
+      i += 1
+    end
+
+    assert_equal 33, i
+    s.override_current_time(s.next_time) # 98
+    assert s.secondary?
+
+    s.step
+    assert_equal s.timeout_at, s.next_time
+    assert s.limit?
+  end
+
+  test 'periodic retries with secondary and specified threshold' do
+    s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true, secondary_threshold: 0.75)
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+    assert_equal (dummy_current_time + 100), s.timeout_at
+    assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at
+  end
+
+  test 'exponential backoff forever without randomization' do
+    s = @d.retry_state_create(:t11, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2)
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+
+    assert_equal 0, s.steps
+    assert_equal (dummy_current_time + 0.1), s.next_time
+
+    i = 1
+    while i < 300
+      s.step
+      assert_equal i, s.steps
+      assert_equal (dummy_current_time + 0.1 * (2 ** (i - 1))), s.next_time
+      assert !s.limit?
+      i += 1
+    end
+  end
+
+  test 'exponential backoff with max_interval' do
+    s = @d.retry_state_create(:t12, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2, max_interval: 100)
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+
+    assert_equal 0, s.steps
+    assert_equal (dummy_current_time + 0.1), s.next_time
+
+    # 0.1 * (2 ** (10 - 1)) == 0.1 * 2 ** 9 == 51.2
+    # 0.1 * (2 ** (11 - 1)) == 0.1 * 2 ** 10 == 102.4
+    i = 1
+    while i < 11
+      s.step
+      assert_equal i, s.steps
+      assert_equal (dummy_current_time + 0.1 * (2 ** (i - 1))), s.next_time, "start:#{dummy_current_time}, i:#{i}"
+      i += 1
+    end
+
+    s.step
+    assert_equal 11, s.steps
+    assert_equal (dummy_current_time + 100), s.next_time
+
+    s.step
+    assert_equal 12, s.steps
+    assert_equal (dummy_current_time + 100), s.next_time
+  end
+
+  test 'exponential backoff with shorter timeout' do
+    s = @d.retry_state_create(:t13, :exponential_backoff, 1, 12, randomize: false, backoff_base: 2, max_interval: 10)
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+
+    assert_equal (dummy_current_time + 12), s.timeout_at
+
+    assert_equal 0, s.steps
+    assert_equal (dummy_current_time + 1), s.next_time
+
+    # 1 + 1 + 2 + 4 (=8)
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 1, s.steps
+    assert_equal (s.current_time + 1), s.next_time
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 2, s.steps
+    assert_equal (s.current_time + 2), s.next_time
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 3, s.steps
+    assert_equal (s.current_time + 4), s.next_time
+
+    assert !s.limit?
+
+    # + 8 (=16) > 12
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 4, s.steps
+    assert_equal s.timeout_at, s.next_time
+
+    assert s.limit?
+  end
+
+  test 'exponential backoff with max_steps' do
+    s = @d.retry_state_create(:t14, :exponential_backoff, 1, 120, randomize: false, backoff_base: 2, max_interval: 10, max_steps: 6)
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+
+    assert_equal (dummy_current_time + 120), s.timeout_at
+
+    assert_equal 0, s.steps
+    assert_equal (dummy_current_time + 1), s.next_time
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 1, s.steps
+    assert_equal (s.current_time + 1), s.next_time
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 2, s.steps
+    assert_equal (s.current_time + 2), s.next_time
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 3, s.steps
+    assert_equal (s.current_time + 4), s.next_time
+
+    assert !s.limit?
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 4, s.steps
+    assert_equal (s.current_time + 8), s.next_time
+
+    assert !s.limit?
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 5, s.steps
+    assert_equal (s.current_time + 10), s.next_time
+
+    assert !s.limit?
+
+    s.override_current_time(s.next_time)
+    s.step
+    assert_equal 6, s.steps
+    assert_equal (s.current_time + 10), s.next_time
+
+    assert s.limit?
+  end
+
+  test 'exponential backoff retries with secondary' do
+    s = @d.retry_state_create(:t15, :exponential_backoff, 1, 100, randomize: false, backoff_base: 2, secondary: true) # threshold 0.8
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+    assert_equal (dummy_current_time + 100), s.timeout_at
+    assert_equal (dummy_current_time + 100 * 0.8), s.secondary_transition_at
+
+    assert_equal (dummy_current_time + 1), s.next_time
+    assert !s.secondary?
+
+    # 1, 1(2), 2(4), 4(8), 8(16), 16(32), 32(64), (80), (81), (83), (87), (95), (100)
+    i = 1
+    while i < 7
+      s.override_current_time(s.next_time)
+      assert !s.secondary?
+
+      s.step
+      assert_equal i, s.steps
+      assert_equal (s.current_time + 1 * (2 ** (i - 1))), s.next_time
+      assert !s.limit?
+      i += 1
+    end
+
+    assert_equal 7, i
+    s.override_current_time(s.next_time) # 64
+    assert !s.secondary?
+
+    s.step
+    assert_equal 7, s.steps
+    assert_equal s.secondary_transition_at, s.next_time
+    assert !s.limit?
+
+    i += 1
+    assert_equal 8, i
+    s.override_current_time(s.next_time) # 80
+    assert s.secondary?
+
+    s.step
+    assert_equal 8, s.steps
+    assert_equal s.steps, s.secondary_transition_steps
+    assert_equal (s.secondary_transition_at + 1.0), s.next_time
+    assert !s.limit?
+
+    # 81, 82, 84, 88, 96, 100
+    j = 1
+    while j < 4
+      s.override_current_time(s.next_time)
+      assert s.secondary?
+      assert_equal :secondary, s.current
+
+      s.step
+      assert_equal (8 + j), s.steps
+      assert_equal (s.current_time + (1 * (2 ** j))), s.next_time
+      assert !s.limit?, "j:#{j}"
+      j += 1
+    end
+
+    assert_equal 4, j
+    s.override_current_time(s.next_time) # 96
+    assert s.secondary?
+
+    s.step
+    assert_equal s.timeout_at, s.next_time
+    assert s.limit?
+  end
+
+  test 'exponential backoff retries with secondary and specified threshold' do
+    s = @d.retry_state_create(:t16, :exponential_backoff, 1, 100, randomize: false, secondary: true, backoff_base: 2, secondary_threshold: 0.75)
+    dummy_current_time = s.start
+    s.override_current_time(dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+    assert_equal (dummy_current_time + 100), s.timeout_at
+    assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at
+  end
+end