fluentd 0.14.3 → 0.14.4

data/test/plugin/test_output_as_buffered.rb

@@ -145,6 +145,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
       logs = @i.log.out.logs.dup
       @i.start
+      @i.after_start
       assert{ logs.select{|log| log.include?('[warn]') }.size == 0 }
     end
 
@@ -154,6 +155,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       logs = @i.log.out.logs.dup
 
       @i.start # this calls `log.reset`... capturing logs about configure must be done before this line
+      @i.after_start
       assert_equal ['key1', 'key2', 'key3', 'key4'], @i.chunk_keys
 
       assert{ logs.select{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') }.size == 1 }
@@ -164,6 +166,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
       logs = @i.log.out.logs.dup
       @i.start # this calls `log.reset`... capturing logs about configure must be done before this line
+      @i.after_start
       assert{ logs.select{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') }.size == 1 }
     end
 
@@ -172,6 +175,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
       logs = @i.log.out.logs.dup
       @i.start
+      @i.after_start
       assert{ logs.select{|log| log.include?('[warn]') }.size == 0 }
     end
   end
@@ -187,6 +191,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))
       @i.start
+      @i.after_start
     end
 
     test '#start does not create enqueue thread, but creates flush threads' do
@@ -289,6 +294,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))
       @i.start
+      @i.after_start
     end
 
     test '#start creates enqueue thread and flush threads' do
@@ -398,6 +404,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))
       @i.start
+      @i.after_start
     end
 
     test '#start does not create enqueue thread, but creates flush threads' do
@@ -491,6 +498,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.start
+      @i.after_start
     end
 
     test '#configure raises config error if timekey is not specified' do
@@ -706,6 +714,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.start
+      @i.after_start
     end
 
     test 'default flush_mode is set to :interval' do
@@ -922,6 +931,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.start
+      @i.after_start
     end
 
     test 'default flush_mode is set to :interval' do
@@ -1134,6 +1144,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.start
+      @i.after_start
 
       assert_equal :interval, @i.instance_eval{ @flush_mode }
     end
@@ -1150,6 +1161,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.start
+      @i.after_start
 
       assert_equal :lazy, @i.instance_eval{ @flush_mode }
     end
@@ -1168,6 +1180,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:delayed)
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.start
+      @i.after_start
     end
 
     test '#format is called for each event streams' do

data/test/plugin/test_output_as_buffered_overflow.rb

@@ -70,6 +70,7 @@ class BufferedOutputOverflowTest < Test::Unit::TestCase
       @i = create_output()
       @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',hash)]))
       @i.start
+      @i.after_start
     end
 
     test '#emit_events raises error when buffer is full' do
@@ -108,6 +109,7 @@ class BufferedOutputOverflowTest < Test::Unit::TestCase
       @i = create_output()
       @i.configure(config_element('ROOT','',{'log_level' => 'debug'},[config_element('buffer','tag',hash)]))
       @i.start
+      @i.after_start
     end
 
     test '#emit_events blocks until any queues are flushed' do
@@ -169,6 +171,7 @@ class BufferedOutputOverflowTest < Test::Unit::TestCase
       @i = create_output()
       @i.configure(config_element('ROOT','',{'log_level' => 'debug'},[config_element('buffer','tag',hash)]))
       @i.start
+      @i.after_start
     end
 
     test '#emit_events will success by dropping oldest chunk' do

data/test/plugin/test_output_as_buffered_retries.rb

@@ -127,6 +127,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
       @i.start
+      @i.after_start
 
       assert_equal :exponential_backoff, @i.buffer_config.retry_type
       assert_equal 1, @i.buffer_config.retry_wait
@@ -160,6 +161,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -201,6 +203,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -255,6 +258,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -344,6 +348,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -424,6 +429,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -469,6 +475,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -563,6 +570,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -660,6 +668,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -730,6 +739,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -797,6 +807,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
       @i.register(:try_write){|chunk| raise "yay, your #write must fail" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -170,6 +170,10 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       i.start
       assert i.secondary.started?
 
+      assert !i.secondary.after_started?
+      i.after_start
+      assert i.secondary.after_started?
+
       assert !i.secondary.stopped?
       i.stop
       assert i.secondary.stopped?
@@ -207,6 +211,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:prefer_delayed_commit){ false }
       @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -271,6 +276,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:prefer_delayed_commit){ false }
       @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -336,6 +342,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:prefer_delayed_commit){ true }
       @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -412,6 +419,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:prefer_delayed_commit){ true }
       @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -489,6 +497,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.secondary.register(:write){|chunk| raise "don't use this" }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -573,6 +582,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:prefer_delayed_commit){ false }
       @i.secondary.register(:write){|chunk| raise "your secondary is also useless." }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -643,6 +653,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:prefer_delayed_commit){ false }
       @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 
@@ -713,6 +724,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:prefer_delayed_commit){ false }
       @i.secondary.register(:write){|chunk| raise "your secondary is also useless." }
       @i.start
+      @i.after_start
 
       @i.interrupt_flushes
 

data/test/plugin/test_output_as_standard.rb

@@ -85,6 +85,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:standard)
       @i.configure(config_element())
       @i.start
+      @i.after_start
 
       m = create_metadata()
       es = test_event_stream
@@ -99,6 +100,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:standard)
       @i.configure(config_element('ROOT','',{"time_as_integer"=>"true"}))
       @i.start
+      @i.after_start
 
       m = create_metadata()
       es = test_event_stream
@@ -115,6 +117,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:standard)
       @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m = create_metadata(tag: "mytag.test")
       es = test_event_stream
@@ -129,6 +132,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:standard)
       @i.configure(config_element('ROOT','',{"time_as_integer"=>"true"},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m = create_metadata(tag: "mytag.test")
       es = test_event_stream
@@ -145,6 +149,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:standard)
       @i.configure(config_element('ROOT','',{},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
       m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
@@ -177,6 +182,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:standard)
       @i.configure(config_element('ROOT','',{"time_as_integer" => "true"},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
       m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
@@ -211,6 +217,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:standard)
       @i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m1 = create_metadata(variables: {key: "my value", name: "moris1"})
       es1 = Fluent::MultiEventStream.new
@@ -238,6 +245,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i = create_output(:standard)
       @i.configure(config_element('ROOT','',{"time_as_integer" => "true"},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m1 = create_metadata(variables: {key: "my value", name: "moris1"})
       es1 = Fluent::MultiEventStream.new
@@ -268,6 +276,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i.register(:format){|tag, time, record| [time, record].to_json }
       @i.configure(config_element())
       @i.start
+      @i.after_start
 
       m = create_metadata()
       es = test_event_stream
@@ -285,6 +294,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i.register(:format){|tag, time, record| [time, record].to_json }
       @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m = create_metadata(tag: "mytag.test")
       es = test_event_stream
@@ -301,6 +311,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i.register(:format){|tag, time, record| [time, record].to_json }
       @i.configure(config_element('ROOT','',{},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
       m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
@@ -336,6 +347,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       @i.register(:format){|tag, time, record| [time, record].to_json }
       @i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
       @i.start
+      @i.after_start
 
       m1 = create_metadata(variables: {key: "my value", name: "moris1"})
       es1 = Fluent::MultiEventStream.new

data/test/plugin_helper/test_compat_parameters.rb

@@ -186,6 +186,7 @@ class CompatParameterTest < Test::Unit::TestCase
       @i = DummyO4.new
       @i.configure(conf)
       @i.start
+      @i.after_start
 
       assert_equal 'file', @i.buffer_config[:@type]
       assert_equal 10, @i.buffer_config.flush_thread_count
@@ -219,6 +220,7 @@ class CompatParameterTest < Test::Unit::TestCase
       @i = DummyO4.new
       @i.configure(conf)
       @i.start
+      @i.after_start
 
       assert_equal 'file', @i.buffer_config[:@type]
       assert_equal 10, @i.buffer_config.flush_thread_count

data/test/plugin_helper/test_timer.rb

@@ -97,4 +97,35 @@ class TimerTest < Test::Unit::TestCase
 
     d1.shutdown; d1.close; d1.terminate
   end
+
+  test 'can run at once' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.timer_running?
+    d1.start
+    assert d1.timer_running?
+
+    waiting_assertion = true
+    waiting_timer = true
+    counter = 0
+    d1.timer_execute(:test, 1, repeat: false) do
+      sleep(0.1) while waiting_assertion
+      counter += 1
+      waiting_timer = false
+    end
+
+    watchers = d1._event_loop.watchers.reject {|w| w.is_a?(Fluent::PluginHelper::EventLoop::DefaultWatcher) }
+    assert_equal(1, watchers.size)
+    assert(watchers.first.attached?)
+
+    waiting_assertion = false
+    sleep(0.1) while waiting_timer
+
+    assert_equal(1, counter)
+    assert_false(watchers.first.attached?)
+    watchers = d1._event_loop.watchers.reject {|w| w.is_a?(Fluent::PluginHelper::EventLoop::DefaultWatcher) }
+    assert_equal(0, watchers.size)
+
+    d1.shutdown; d1.close; d1.terminate
+  end
 end

data/test/test_event_router.rb

@@ -185,6 +185,10 @@ class EventRouterTest < ::Test::Unit::TestCase
 
     sub_test_case 'filter' do
       test 'filter should be called when tag matched' do
+        filter = Class.new(FluentTestFilter) { |x|
+          def filter_stream(_tag, es); end
+        }.new
+
         event_router.add_rule('test', filter)
 
         assert_rr do
@@ -229,6 +233,89 @@ class EventRouterTest < ::Test::Unit::TestCase
       end
     end
 
+    sub_test_case 'optimized filter' do
+      setup do
+        @record = { 'k' => 'v' }
+        @now = Engine.now
+      end
+
+      test 'call optimized filter when the filter plugin implements #filter without #filter_stream' do
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter).filter('test', @now, @record) { @record }
+          event_router.emit('test', @now, @record)
+        end
+      end
+
+      test 'call optimized filter when the filter plugin implements #filter_with_time without #filter_stream' do
+        filter = Class.new(FluentTestFilter) {
+          undef_method :filter
+          def filter_with_time(tag, time, record); end
+        }.new
+
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter).filter_with_time('test', @now, @record) { [time, @record] }
+          event_router.emit('test', @now, @record)
+        end
+      end
+
+      test "don't call optimized filter when filter plugins implement #filter_stream" do
+        filter = Class.new(FluentTestFilter) {
+          undef_method :filter
+          def filter_stream(tag, time, record); end
+        }.new
+
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter).filter_stream('test', is_a(OneEventStream)) { OneEventStream.new(@now, @record) }
+          event_router.emit('test', @now, @record)
+        end
+      end
+
+      test 'call optimized filter when filter plugins have #filter_with_time instead of #filter' do
+        filter_with_time = Class.new(FluentTestFilter) {
+          undef_method :filter
+          def filter_with_time(tag, time, record); end
+        }.new
+
+        event_router.add_rule('test', filter_with_time)
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter_with_time).filter_with_time('test', @now, @record) { [@now + 1, @record] }
+          mock(filter).filter('test', @now + 1, @record) { @record }
+          event_router.emit('test', @now, @record)
+        end
+      end
+
+      test "don't call optimized filter even if just a filter of some filters implements #filter_stream method" do
+        filter_stream = Class.new(FluentTestFilter) {
+          def filter_stream(tag, record); end
+        }.new
+
+        filter_with_time = Class.new(FluentTestFilter) {
+          undef_method :filter
+          def filter_with_time(tag, time, record); end
+        }.new
+
+        filters = [filter_stream, filter_with_time, filter]
+        filters.each { |f| event_router.add_rule('test', f) }
+
+        e = OneEventStream.new(@now, @record)
+        assert_rr do
+          mock($log).info("Filtering works with worse performance, because #{[filter_stream].map(&:class)} uses `#filter_stream` method.")
+          mock(filter_stream).filter_stream('test', is_a(OneEventStream)) { e }
+          mock(filter).filter_stream('test', is_a(OneEventStream)) { e }
+          mock(filter_with_time).filter_stream('test', is_a(OneEventStream)) { e }
+          event_router.emit('test', @now, @record)
+        end
+      end
+    end
+
     sub_test_case 'emit_error_handler' do
       test 'call handle_emits_error when emit failed' do
         event_router.add_rule('test', error_output)