fluentd 0.14.0 → 0.14.1

data/test/plugin_helper/test_retry_state.rb

@@ -4,16 +4,16 @@ require 'fluent/plugin/base'
 
 require 'time'
 
-class Fluent::PluginHelper::RetryState::RetryStateMachine
-  def override_current_time(time)
-    (class << self; self; end).module_eval do
-      alias_method(:current_time_orig, :current_time)
+class RetryStateHelperTest < Test::Unit::TestCase
+  def override_current_time(state, time)
+    mojule = Module.new do
       define_method(:current_time){ time }
     end
+    state.singleton_class.module_eval do
+      prepend mojule
+    end
   end
-end
 
-class RetryStateHelperTest < Test::Unit::TestCase
   class Dummy < Fluent::Plugin::TestBase
     helpers :retry_state
   end
@@ -55,7 +55,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'periodic retries' do
     s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false)
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
     assert_equal (dummy_current_time + 29), s.timeout_at
@@ -63,7 +63,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
 
     i = 1
     while i < 9
-      s.override_current_time(s.next_time)
+      override_current_time(s, s.next_time)
       s.step
       assert_equal i, s.steps
       assert_equal (s.current_time + 3), s.next_time
@@ -72,7 +72,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     end
 
     assert_equal 9, i
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal s.timeout_at, s.next_time
     assert s.limit?
@@ -81,7 +81,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'periodic retries with max_steps' do
     s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false, max_steps: 5)
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
     assert_equal (dummy_current_time + 29), s.timeout_at
@@ -89,7 +89,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
 
     i = 1
     while i < 5
-      s.override_current_time(s.next_time)
+      override_current_time(s, s.next_time)
       s.step
       assert_equal i, s.steps
       assert_equal (s.current_time + 3), s.next_time
@@ -98,7 +98,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     end
 
     assert_equal 5, i
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal (s.current_time + 3), s.next_time
     assert s.limit?
@@ -107,7 +107,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'periodic retries with secondary' do
     s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true) # threshold 0.8
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
     assert_equal (dummy_current_time + 100), s.timeout_at
@@ -118,7 +118,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
 
     i = 1
     while i < 26
-      s.override_current_time(s.next_time)
+      override_current_time(s, s.next_time)
       assert !s.secondary?
 
       s.step
@@ -129,7 +129,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     end
 
     assert_equal 26, i
-    s.override_current_time(s.next_time) # 78
+    override_current_time(s, s.next_time) # 78
     assert !s.secondary?
 
     s.step
@@ -139,7 +139,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
 
     i += 1
     assert_equal 27, i
-    s.override_current_time(s.next_time) # 80
+    override_current_time(s, s.next_time) # 80
     assert s.secondary?
 
     s.step
@@ -150,7 +150,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     i += 1
 
     while i < 33
-      s.override_current_time(s.next_time)
+      override_current_time(s, s.next_time)
       assert s.secondary?
 
       s.step
@@ -160,7 +160,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     end
 
     assert_equal 33, i
-    s.override_current_time(s.next_time) # 98
+    override_current_time(s, s.next_time) # 98
     assert s.secondary?
 
     s.step
@@ -171,7 +171,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'periodic retries with secondary and specified threshold' do
     s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true, secondary_threshold: 0.75)
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
     assert_equal (dummy_current_time + 100), s.timeout_at
@@ -181,7 +181,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'exponential backoff forever without randomization' do
     s = @d.retry_state_create(:t11, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2)
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
 
@@ -201,7 +201,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'exponential backoff with max_interval' do
     s = @d.retry_state_create(:t12, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2, max_interval: 100)
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
 
@@ -230,7 +230,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'exponential backoff with shorter timeout' do
     s = @d.retry_state_create(:t13, :exponential_backoff, 1, 12, randomize: false, backoff_base: 2, max_interval: 10)
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
 
@@ -241,17 +241,17 @@ class RetryStateHelperTest < Test::Unit::TestCase
 
     # 1 + 1 + 2 + 4 (=8)
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 1, s.steps
     assert_equal (s.current_time + 1), s.next_time
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 2, s.steps
     assert_equal (s.current_time + 2), s.next_time
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 3, s.steps
     assert_equal (s.current_time + 4), s.next_time
@@ -260,7 +260,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
 
     # + 8 (=16) > 12
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 4, s.steps
     assert_equal s.timeout_at, s.next_time
@@ -271,7 +271,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'exponential backoff with max_steps' do
     s = @d.retry_state_create(:t14, :exponential_backoff, 1, 120, randomize: false, backoff_base: 2, max_interval: 10, max_steps: 6)
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
 
@@ -280,38 +280,38 @@ class RetryStateHelperTest < Test::Unit::TestCase
     assert_equal 0, s.steps
     assert_equal (dummy_current_time + 1), s.next_time
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 1, s.steps
     assert_equal (s.current_time + 1), s.next_time
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 2, s.steps
     assert_equal (s.current_time + 2), s.next_time
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 3, s.steps
     assert_equal (s.current_time + 4), s.next_time
 
     assert !s.limit?
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 4, s.steps
     assert_equal (s.current_time + 8), s.next_time
 
     assert !s.limit?
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 5, s.steps
     assert_equal (s.current_time + 10), s.next_time
 
     assert !s.limit?
 
-    s.override_current_time(s.next_time)
+    override_current_time(s, s.next_time)
     s.step
     assert_equal 6, s.steps
     assert_equal (s.current_time + 10), s.next_time
@@ -322,7 +322,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'exponential backoff retries with secondary' do
     s = @d.retry_state_create(:t15, :exponential_backoff, 1, 100, randomize: false, backoff_base: 2, secondary: true) # threshold 0.8
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
     assert_equal (dummy_current_time + 100), s.timeout_at
@@ -334,7 +334,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     # 1, 1(2), 2(4), 4(8), 8(16), 16(32), 32(64), (80), (81), (83), (87), (95), (100)
     i = 1
     while i < 7
-      s.override_current_time(s.next_time)
+      override_current_time(s, s.next_time)
       assert !s.secondary?
 
       s.step
@@ -345,7 +345,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     end
 
     assert_equal 7, i
-    s.override_current_time(s.next_time) # 64
+    override_current_time(s, s.next_time) # 64
     assert !s.secondary?
 
     s.step
@@ -355,7 +355,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
 
     i += 1
     assert_equal 8, i
-    s.override_current_time(s.next_time) # 80
+    override_current_time(s, s.next_time) # 80
     assert s.secondary?
 
     s.step
@@ -367,7 +367,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     # 81, 82, 84, 88, 96, 100
     j = 1
     while j < 4
-      s.override_current_time(s.next_time)
+      override_current_time(s, s.next_time)
       assert s.secondary?
       assert_equal :secondary, s.current
 
@@ -379,7 +379,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
     end
 
     assert_equal 4, j
-    s.override_current_time(s.next_time) # 96
+    override_current_time(s, s.next_time) # 96
     assert s.secondary?
 
     s.step
@@ -390,7 +390,7 @@ class RetryStateHelperTest < Test::Unit::TestCase
   test 'exponential backoff retries with secondary and specified threshold' do
     s = @d.retry_state_create(:t16, :exponential_backoff, 1, 100, randomize: false, secondary: true, backoff_base: 2, secondary_threshold: 0.75)
     dummy_current_time = s.start
-    s.override_current_time(dummy_current_time)
+    override_current_time(s, dummy_current_time)
 
     assert_equal dummy_current_time, s.current_time
     assert_equal (dummy_current_time + 100), s.timeout_at

data/test/plugin_helper/test_storage.rb

@@ -169,16 +169,6 @@ class StorageHelperTest < Test::Unit::TestCase
     assert{ s.implementation.is_a? ExampleStorage }
   end
 
-  test 'raises exception for storage creation without explicit type specification' do
-    @d = d = Dummy.new
-    d.configure(config_element())
-    d.start
-
-    assert_raises ArgumentError do
-      d.storage_create(usage: 'mydata', conf: config_element('storage', 'mydata', {'@type' => 'example'}))
-    end
-  end
-
   test 'creates 2 or more storage plugin instances' do
     @d = d = Dummy.new
     conf = config_element('ROOT', '', {}, [
@@ -369,6 +359,8 @@ class StorageHelperTest < Test::Unit::TestCase
     assert_equal 1, s.implementation.load_times
     assert_equal 0, s.implementation.save_times
 
+    d.before_shutdown
+
     d.shutdown
 
     assert_equal 1, s.implementation.load_times
@@ -408,4 +400,79 @@ class StorageHelperTest < Test::Unit::TestCase
     assert_equal 0, s.implementation.data.size
     assert_equal 0, s.implementation.saved.size
   end
+
+  test 'calls lifecycle methods for all plugin instances via owner plugin' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [ config_element('storage', '', {'@type' => 'example'}), config_element('storage', 'e2', {'@type' => 'example'}) ])
+    d.configure(conf)
+    d.start
+
+    i1 = d.storage_create(usage: '')
+    i2 = d.storage_create(usage: 'e2')
+    i3 = d.storage_create(usage: 'e3', type: 'ex2')
+
+    assert i1.started?
+    assert i2.started?
+    assert i3.started?
+
+    assert !i1.stopped?
+    assert !i2.stopped?
+    assert !i3.stopped?
+
+    d.stop
+
+    assert i1.stopped?
+    assert i2.stopped?
+    assert i3.stopped?
+
+    assert !i1.before_shutdown?
+    assert !i2.before_shutdown?
+    assert !i3.before_shutdown?
+
+    d.before_shutdown
+
+    assert i1.before_shutdown?
+    assert i2.before_shutdown?
+    assert i3.before_shutdown?
+
+    assert !i1.shutdown?
+    assert !i2.shutdown?
+    assert !i3.shutdown?
+
+    d.shutdown
+
+    assert i1.shutdown?
+    assert i2.shutdown?
+    assert i3.shutdown?
+
+    assert !i1.after_shutdown?
+    assert !i2.after_shutdown?
+    assert !i3.after_shutdown?
+
+    d.after_shutdown
+
+    assert i1.after_shutdown?
+    assert i2.after_shutdown?
+    assert i3.after_shutdown?
+
+    assert !i1.closed?
+    assert !i2.closed?
+    assert !i3.closed?
+
+    d.close
+
+    assert i1.closed?
+    assert i2.closed?
+    assert i3.closed?
+
+    assert !i1.terminated?
+    assert !i2.terminated?
+    assert !i3.terminated?
+
+    d.terminate
+
+    assert i1.terminated?
+    assert i2.terminated?
+    assert i3.terminated?
+  end
 end

data/test/scripts/fluent/plugin/out_test.rb

@@ -14,17 +14,25 @@
 #    limitations under the License.
 #
 
-module Fluent
+require 'fluent/plugin/output'
+require 'fluent/event'
+
+module Fluent::Plugin
   class TestOutput < Output
-    Plugin.register_output('test', self)
+    Fluent::Plugin.register_output('test', self)
+
+    config_param :name, :string
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+    end
 
     def initialize
       super
       @emit_streams = []
-      @name = nil
     end
 
-    attr_reader :emit_streams, :name
+    attr_reader :emit_streams
 
     def emits
       all = []
@@ -54,23 +62,20 @@ module Fluent
       all
     end
 
-    def configure(conf)
-      if name = conf['name']
-        @name = name
-      end
-    end
-
-    def start
-      super
+    def prefer_buffered_processing
+      false
     end
 
-    def shutdown
-      super
+    def process(tag, es)
+      @emit_streams << [tag, es.to_a]
     end
 
-    def emit(tag, es, chain)
-      chain.next
-      @emit_streams << [tag, es.to_a]
+    def write(chunk)
+      es = Fluent::ArrayEventStream.new
+      chunk.each do |time, record|
+        es.add(time, record)
+      end
+      @emit_streams << [tag, es]
     end
   end
 end