fluentd 0.12.40 → 1.6.2

data/test/test_event.rb

@@ -1,17 +1,45 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/event'
+require 'fluent/plugin/compressable'
 
 module EventTest
+  module DeepCopyAssertion
+    def assert_duplicated_records(es1, es2)
+      ary1 = []
+      es1.each do |_, record|
+        ary1 << record
+      end
+      ary2 = []
+      es2.each do |_, record|
+        ary2 << record
+      end
+      assert_equal ary1.size, ary2.size
+      ary1.each_with_index do |r, i|
+        assert_not_equal r.object_id, ary2[i].object_id
+      end
+    end
+  end
+
   class OneEventStreamTest < ::Test::Unit::TestCase
     include Fluent
+    include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
 
     def setup
-      @time = Engine.now
+      @time = event_time()
       @record = {'k' => 'v', 'n' => 1}
       @es = OneEventStream.new(@time, @record)
     end
 
+    test 'empty?' do
+      assert_false @es.empty?
+    end
+
+    test 'size' do
+      assert_equal 1, @es.size
+    end
+
     test 'repeatable?' do
       assert_true @es.repeatable?
     end
@@ -20,6 +48,21 @@ module EventTest
       dupped = @es.dup
       assert_kind_of OneEventStream, dupped
       assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+    end
+
+    test 'slice' do
+      assert_equal 0, @es.slice(1, 1).size
+      assert_equal 0, @es.slice(0, 0).size
+
+      sliced = @es.slice(0, 1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @time, time
+        assert_equal @record, record
+      end
     end
 
     test 'each' do
@@ -31,18 +74,45 @@ module EventTest
 
     test 'to_msgpack_stream' do
       stream = @es.to_msgpack_stream
-      MessagePack::Unpacker.new.feed_each(stream) { |time, record|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
         assert_equal @time, time
         assert_equal @record, record
       }
     end
+
+    test 'to_msgpack_stream with time_int argument' do
+      stream = @es.to_msgpack_stream(time_int: true)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @time.to_i, time
+        assert_equal @record, record
+      }
+    end
+
+    test 'to_compressed_msgpack_stream' do
+      stream = @es.to_compressed_msgpack_stream
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(decompress(stream)) { |time, record|
+        assert_equal @time, time
+        assert_equal @record, record
+      }
+    end
+
+    test 'to_compressed_msgpack_stream with time_int argument' do
+      stream = @es.to_compressed_msgpack_stream(time_int: true)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(decompress(stream)) { |time, record|
+        assert_equal @time.to_i, time
+        assert_equal @record, record
+      }
+    end
   end
 
   class ArrayEventStreamTest < ::Test::Unit::TestCase
     include Fluent
+    include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
 
     def setup
-      @times = [Engine.now, Engine.now + 1]
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
       @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
       @es = ArrayEventStream.new(@times.zip(@records))
     end
@@ -55,6 +125,7 @@ module EventTest
       dupped = @es.dup
       assert_kind_of ArrayEventStream, dupped
       assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
     end
 
     test 'empty?' do
@@ -62,6 +133,35 @@ module EventTest
       assert_true ArrayEventStream.new([]).empty?
     end
 
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, ArrayEventStream.new([]).size
+    end
+
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
     test 'each' do
       i = 0
       @es.each { |time, record|
@@ -74,23 +174,48 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      MessagePack::Unpacker.new.feed_each(stream) { |time, record|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
       }
     end
+
+    test 'to_compressed_msgpack_stream' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_compressed_msgpack_stream with time_int argument' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream(time_int: true)
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i].to_i, time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
   end
 
   class MultiEventStreamTest < ::Test::Unit::TestCase
     include Fluent
+    include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
 
     def setup
-      @times = [Engine.now, Engine.now + 1]
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
       @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
       @es = MultiEventStream.new
-      @times.zip(@records).each { |time, record|
-        @es.add(time, record)
+      @times.zip(@records).each { |_time, record|
+        @es.add(_time, record)
       }
     end
 
@@ -102,6 +227,7 @@ module EventTest
       dupped = @es.dup
       assert_kind_of MultiEventStream, dupped
       assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
     end
 
     test 'empty?' do
@@ -109,6 +235,35 @@ module EventTest
       assert_true MultiEventStream.new.empty?
     end
 
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, MultiEventStream.new.size
+    end
+
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
     test 'each' do
       i = 0
       @es.each { |time, record|
@@ -121,31 +276,102 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      MessagePack::Unpacker.new.feed_each(stream) { |time, record|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_compressed_msgpack_stream' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
       }
     end
+
+    test 'to_compressed_msgpack_stream with time_int argument' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream(time_int: true)
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i].to_i, time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
   end
 
   class MessagePackEventStreamTest < ::Test::Unit::TestCase
     include Fluent
+    include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
 
     def setup
-      pk = MessagePack::Packer.new
-      @times = [Engine.now, Engine.now + 1]
+      pk = Fluent::Engine.msgpack_factory.packer
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
       @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
-      @times.zip(@records).each { |time, record|
-        pk.write([time, record])
+      @times.zip(@records).each { |_time, record|
+        pk.write([_time, record])
       }
       @es = MessagePackEventStream.new(pk.to_s)
     end
 
+    test 'dup' do
+      dupped = @es.dup
+      assert_kind_of MessagePackEventStream, dupped
+      assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+
+      # After iteration of events (done in assert_duplicated_records),
+      # duplicated event stream still has unpacked objects and correct size
+      dupped = @es.dup
+      assert_equal 2, dupped.instance_eval{ @size }
+    end
+
+    test 'empty?' do
+      assert_false @es.empty?
+      assert_true MessagePackEventStream.new('', 0).empty?
+    end
+
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, MessagePackEventStream.new('').size
+    end
+
     test 'repeatable?' do
       assert_true @es.repeatable?
     end
 
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
     test 'each' do
       i = 0
       @es.each { |time, record|
@@ -158,7 +384,128 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      MessagePack::Unpacker.new.feed_each(stream) { |time, record|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_compressed_msgpack_stream' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+  end
+
+  class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase
+    include Fluent
+    include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
+
+    def setup
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
+      @records = [{ 'k' => 'v1', 'n' => 1 }, { 'k' => 'v2', 'n' => 2 }]
+      @packed_record = ''
+      @entries = ''
+      @times.zip(@records).each do |_time, record|
+        v = [_time, record].to_msgpack
+        @packed_record += v
+        @entries += compress(v)
+      end
+      @es = CompressedMessagePackEventStream.new(@entries)
+    end
+
+    def ensure_data_is_decompressed
+      assert_equal @entries, @es.instance_variable_get(:@data)
+      yield
+      assert_equal @packed_record, @es.instance_variable_get(:@data)
+    end
+
+    test 'dup' do
+      dupped = @es.dup
+      assert_kind_of CompressedMessagePackEventStream, dupped
+      assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+
+      # After iteration of events (done in assert_duplicated_records),
+      # duplicated event stream still has unpacked objects and correct size
+      dupped = @es.dup
+      assert_equal 2, dupped.instance_eval{ @size }
+    end
+
+    test 'repeatable?' do
+      assert_true @es.repeatable?
+    end
+
+    test 'size' do
+      assert_equal 0, CompressedMessagePackEventStream.new('').size
+      ensure_data_is_decompressed { assert_equal 2, @es.size  }
+    end
+
+    test 'each' do
+      i = 0
+      ensure_data_is_decompressed do
+        @es.each do |time, record|
+          assert_equal @times[i], time
+          assert_equal @records[i], record
+          i += 1
+        end
+      end
+    end
+
+    test 'slice' do
+      sliced = nil
+      ensure_data_is_decompressed { sliced = @es.slice(1,1) }
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
+    test 'to_msgpack_stream' do
+      i = 0
+      stream = nil
+      ensure_data_is_decompressed { stream = @es.to_msgpack_stream }
+
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_compressed_msgpack_stream' do
+      i = 0
+      # Do not call ensure_decompressed!
+      assert_equal @entries, @es.instance_variable_get(:@data)
+      compressed_stream = @es.to_compressed_msgpack_stream
+      assert_equal @entries, @es.instance_variable_get(:@data)
+
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1

data/test/test_event_router.rb

@@ -9,6 +9,7 @@ class EventRouterTest < ::Test::Unit::TestCase
   teardown do
     @output = nil
     @filter = nil
+    @compat_filter = nil
     @error_output = nil
     @emit_handler = nil
     @default_collector = nil
@@ -22,6 +23,10 @@ class EventRouterTest < ::Test::Unit::TestCase
     @filter ||= FluentTestFilter.new
   end
 
+  def compat_filter
+    @compat_filter ||= FluentCompatTestFilter.new
+  end
+
   def error_output
     @error_output ||= FluentTestErrorOutput.new
   end
@@ -97,7 +102,7 @@ class EventRouterTest < ::Test::Unit::TestCase
 
     test 'set one output' do
       @pipeline.set_output(output)
-      @pipeline.emit('test', @es, nil)
+      @pipeline.emit_events('test', @es)
       assert_equal 1, output.events.size
       assert_equal 'value', output.events['test'].first['key']
     end
@@ -107,17 +112,21 @@ class EventRouterTest < ::Test::Unit::TestCase
         @pipeline.set_output(output)
       end
 
-      test 'set one filer' do
-        @pipeline.add_filter(filter)
-        @pipeline.emit('test', @es, nil)
+      data('Filter plugin' => 'filter',
+           'Compat::Filter plugin' => 'compat_filter')
+      test 'set one filer' do |filter_type|
+        @pipeline.add_filter(filter_type == 'filter' ? filter : compat_filter)
+        @pipeline.emit_events('test', @es)
         assert_equal 1, output.events.size
         assert_equal 'value', output.events['test'].first['key']
         assert_equal 0, output.events['test'].first['__test__']
       end
 
-      test 'set one filer with multi events' do
-        @pipeline.add_filter(filter)
-        @pipeline.emit('test', events, nil)
+      data('Filter plugin' => 'filter',
+           'Compat::Filter plugin' => 'compat_filter')
+      test 'set one filer with multi events' do |filter_type|
+        @pipeline.add_filter(filter_type == 'filter' ? filter : compat_filter)
+        @pipeline.emit_events('test', events)
         assert_equal 1, output.events.size
         assert_equal 5, output.events['test'].size
         DEFAULT_EVENT_NUM.times { |i|
@@ -140,7 +149,7 @@ class EventRouterTest < ::Test::Unit::TestCase
     sub_test_case 'default collector' do
       test 'call default collector when no output' do
         assert_rr do
-          mock(default_collector).emit('test', is_a(OneEventStream), NullOutputChain.instance)
+          mock(default_collector).emit_events('test', is_a(OneEventStream))
           event_router.emit('test', Engine.now, 'k' => 'v')
         end
       end
@@ -149,7 +158,7 @@ class EventRouterTest < ::Test::Unit::TestCase
         event_router.add_rule('test', filter)
         assert_rr do
           # After apply Filter, EventStream becomes MultiEventStream by default
-          mock(default_collector).emit('test', is_a(MultiEventStream), NullOutputChain.instance)
+          mock(default_collector).emit_events('test', is_a(MultiEventStream))
           event_router.emit('test', Engine.now, 'k' => 'v')
         end
         assert_equal 1, filter.num
@@ -158,7 +167,7 @@ class EventRouterTest < ::Test::Unit::TestCase
       test "call default collector when no matched with output" do
         event_router.add_rule('test', output)
         assert_rr do
-          mock(default_collector).emit('dummy', is_a(OneEventStream), NullOutputChain.instance)
+          mock(default_collector).emit_events('dummy', is_a(OneEventStream))
           event_router.emit('dummy', Engine.now, 'k' => 'v')
         end
       end
@@ -166,7 +175,7 @@ class EventRouterTest < ::Test::Unit::TestCase
       test "don't call default collector when tag matched" do
         event_router.add_rule('test', output)
         assert_rr do
-          dont_allow(default_collector).emit('test', is_a(OneEventStream), NullOutputChain.instance)
+          dont_allow(default_collector).emit_events('test', is_a(OneEventStream))
           event_router.emit('test', Engine.now, 'k' => 'v')
         end
         # check emit handler doesn't catch rr error
@@ -176,6 +185,10 @@ class EventRouterTest < ::Test::Unit::TestCase
 
     sub_test_case 'filter' do
       test 'filter should be called when tag matched' do
+        filter = Class.new(FluentTestFilter) { |x|
+          def filter_stream(_tag, es); end
+        }.new
+
         event_router.add_rule('test', filter)
 
         assert_rr do
@@ -220,10 +233,94 @@ class EventRouterTest < ::Test::Unit::TestCase
       end
     end
 
+    sub_test_case 'optimized filter' do
+      setup do
+        @record = { 'k' => 'v' }
+        @now = Engine.now
+      end
+
+      test 'call optimized filter when the filter plugin implements #filter without #filter_stream' do
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter).filter('test', @now, @record) { @record }
+          event_router.emit('test', @now, @record)
+        end
+      end
+
+      test 'call optimized filter when the filter plugin implements #filter_with_time without #filter_stream' do
+        filter = Class.new(FluentTestFilter) {
+          undef_method :filter
+          def filter_with_time(tag, time, record); end
+        }.new
+
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter).filter_with_time('test', @now, @record) { [time, @record] }
+          event_router.emit('test', @now, @record)
+        end
+      end
+
+      test "don't call optimized filter when filter plugins implement #filter_stream" do
+        filter = Class.new(FluentTestFilter) {
+          undef_method :filter
+          def filter_stream(tag, time, record); end
+        }.new
+
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter).filter_stream('test', is_a(OneEventStream)) { OneEventStream.new(@now, @record) }
+          event_router.emit('test', @now, @record)
+        end
+      end
+
+      test 'call optimized filter when filter plugins have #filter_with_time instead of #filter' do
+        filter_with_time = Class.new(FluentTestFilter) {
+          undef_method :filter
+          def filter_with_time(tag, time, record); end
+        }.new
+
+        event_router.add_rule('test', filter_with_time)
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter_with_time).filter_with_time('test', @now, @record) { [@now + 1, @record] }
+          mock(filter).filter('test', @now + 1, @record) { @record }
+          event_router.emit('test', @now, @record)
+        end
+      end
+
+      test "don't call optimized filter even if just a filter of some filters implements #filter_stream method" do
+        filter_stream = Class.new(FluentTestFilter) {
+          def filter_stream(tag, record); end
+        }.new
+
+        filter_with_time = Class.new(FluentTestFilter) {
+          undef_method :filter
+          def filter_with_time(tag, time, record); end
+        }.new
+
+        filters = [filter_stream, filter_with_time, filter]
+        filters.each { |f| event_router.add_rule('test', f) }
+
+        e = OneEventStream.new(@now, @record)
+        assert_rr do
+          mock($log).info("disable filter chain optimization because #{[filter_stream].map(&:class)} uses `#filter_stream` method.")
+          mock(filter_stream).filter_stream('test', is_a(OneEventStream)) { e }
+          mock(filter).filter_stream('test', is_a(OneEventStream)) { e }
+          mock(filter_with_time).filter_stream('test', is_a(OneEventStream)) { e }
+          event_router.emit('test', @now, @record)
+        end
+      end
+    end
+
     sub_test_case 'emit_error_handler' do
       test 'call handle_emits_error when emit failed' do
         event_router.add_rule('test', error_output)
 
+        event_router.emit('test', Engine.now, 'k' => 'v')
         assert_rr do
           mock(emit_handler).handle_emits_error('test', is_a(OneEventStream), is_a(RuntimeError))
           event_router.emit('test', Engine.now, 'k' => 'v')