fluentd 0.14.1 → 0.14.2

data/test/plugin_helper/test_storage.rb

@@ -73,6 +73,14 @@ class StorageHelperTest < Test::Unit::TestCase
     helpers :storage
   end
 
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :storage
+    config_section :storage do
+      config_set_default :@type, 'ex2'
+      config_set_default :dummy_path, '/tmp/yay'
+    end
+  end
+
   setup do
     @d = nil
   end
@@ -91,6 +99,41 @@ class StorageHelperTest < Test::Unit::TestCase
     assert_equal 0, d._storages.size
   end
 
+  test 'can override default configuration parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    d.configure(config_element())
+    assert_equal [], d.storage_configs
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {}, [])]))
+    assert_raise NoMethodError do
+      d.storage
+    end
+    assert_equal 1, d.storage_configs.size
+    assert_equal 'ex2', d.storage_configs.first[:@type]
+    assert_equal '/tmp/yay', d.storage_configs.first.dummy_path
+  end
+
+  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.storage_create(conf: config_element('format', '', {'@type' => 'example'}), default_type: 'ex2')
+    assert{ i.is_a?(Fluent::PluginHelper::Storage::SynchronizeWrapper) && i.instance_eval{ @storage }.is_a?(ExampleStorage) }
+
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.storage_create(conf: nil, default_type: 'ex2')
+    assert{ i.is_a?(Fluent::PluginHelper::Storage::SynchronizeWrapper) && i.instance_eval{ @storage }.is_a?(Example2Storage) }
+  end
+
+  test 'raises config error if config section is specified, but @type is not specified' do
+    d = Dummy2.new
+    d.configure(config_element())
+    assert_raise Fluent::ConfigError.new("@type is required in <storage>") do
+      d.storage_create(conf: config_element('storage', '', {}), default_type: 'ex2')
+    end
+  end
+
   test 'can be configured without storage sections' do
     d = Dummy.new
     assert_nothing_raised do

data/test/test_event.rb

@@ -114,6 +114,35 @@ module EventTest
       assert_true ArrayEventStream.new([]).empty?
     end
 
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, ArrayEventStream.new([]).size
+    end
+
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
     test 'each' do
       i = 0
       @es.each { |time, record|
@@ -164,6 +193,35 @@ module EventTest
       assert_true MultiEventStream.new.empty?
     end
 
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, MultiEventStream.new.size
+    end
+
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
     test 'each' do
       i = 0
       @es.each { |time, record|
@@ -204,16 +262,51 @@ module EventTest
       assert_kind_of MessagePackEventStream, dupped
       assert_not_equal @es.object_id, dupped.object_id
       assert_duplicated_records @es, dupped
+
+      # After iteration of events (done in assert_duplicated_records),
+      # duplicated event stream still has unpacked objects and correct size
+      dupped = @es.dup
+      assert_equal 2, dupped.instance_eval{ @size }
     end
 
     test 'empty?' do
       assert_false @es.empty?
+      assert_true MessagePackEventStream.new('', 0).empty?
+    end
+
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, MessagePackEventStream.new('').size
     end
 
     test 'repeatable?' do
       assert_true @es.repeatable?
     end
 
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
     test 'each' do
       i = 0
       @es.each { |time, record|

data/test/test_event_router.rb

@@ -9,6 +9,7 @@ class EventRouterTest < ::Test::Unit::TestCase
   teardown do
     @output = nil
     @filter = nil
+    @compat_filter = nil
     @error_output = nil
     @emit_handler = nil
     @default_collector = nil
@@ -22,6 +23,10 @@ class EventRouterTest < ::Test::Unit::TestCase
     @filter ||= FluentTestFilter.new
   end
 
+  def compat_filter
+    @compat_filter ||= FluentCompatTestFilter.new
+  end
+
   def error_output
     @error_output ||= FluentTestErrorOutput.new
   end
@@ -107,16 +112,20 @@ class EventRouterTest < ::Test::Unit::TestCase
         @pipeline.set_output(output)
       end
 
-      test 'set one filer' do
-        @pipeline.add_filter(filter)
+      data('Filter plugin' => 'filter',
+           'Compat::Filter plugin' => 'compat_filter')
+      test 'set one filer' do |filter_type|
+        @pipeline.add_filter(filter_type == 'filter' ? filter : compat_filter)
         @pipeline.emit_events('test', @es)
         assert_equal 1, output.events.size
         assert_equal 'value', output.events['test'].first['key']
         assert_equal 0, output.events['test'].first['__test__']
       end
 
-      test 'set one filer with multi events' do
-        @pipeline.add_filter(filter)
+      data('Filter plugin' => 'filter',
+           'Compat::Filter plugin' => 'compat_filter')
+      test 'set one filer with multi events' do |filter_type|
+        @pipeline.add_filter(filter_type == 'filter' ? filter : compat_filter)
         @pipeline.emit_events('test', events)
         assert_equal 1, output.events.size
         assert_equal 5, output.events['test'].size

data/test/test_event_time.rb

@@ -45,21 +45,18 @@ class EventTimeTest < Test::Unit::TestCase
 
   test 'JSON.dump' do
     time = Fluent::EventTime.new(100)
-    assert_equal('100', JSON.dump(time))
     assert_equal('{"time":100}', JSON.dump({'time' => time}))
     assert_equal('["tag",100,{"key":"value"}]', JSON.dump(["tag", time, {"key" => "value"}]))
   end
 
   test 'Oj.dump' do
     time = Fluent::EventTime.new(100)
-    assert_equal('100', Oj.dump(time, mode: :compat))
     assert_equal('{"time":100}', Oj.dump({'time' => time}, mode: :compat))
     assert_equal('["tag",100,{"key":"value"}]', Oj.dump(["tag", time, {"key" => "value"}], mode: :compat))
   end
 
   test 'Yajl.dump' do
     time = Fluent::EventTime.new(100)
-    assert_equal('100', Yajl.dump(time))
     assert_equal('{"time":100}', Yajl.dump({'time' => time}))
     assert_equal('["tag",100,{"key":"value"}]', Yajl.dump(["tag", time, {"key" => "value"}]))
   end

data/test/test_formatter.rb

@@ -5,28 +5,12 @@ require 'fluent/formatter'
 module FormatterTest
   include Fluent
 
-  def time2str(time, localtime = false, format = nil)
-    if format
-      if localtime
-        Time.at(time).strftime(format)
-      else
-        Time.at(time).utc.strftime(format)
-      end
-    else
-      if localtime
-        Time.at(time).iso8601
-      else
-        Time.at(time).utc.iso8601
-      end
-    end
-  end
-
   def tag
     'tag'
   end
 
   def record
-    {'message' => 'awesome'}
+    {'message' => 'awesome', 'greeting' => 'hello'}
   end
 
   def with_timezone(tz)
@@ -137,36 +121,6 @@ module FormatterTest
 
       assert_equal("#{Yajl.dump(record)}\n", formatted)
     end
-
-    data('oj' => 'oj', 'yajl' => 'yajl')
-    def test_format_with_include_tag(data)
-      @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo', 'json_parser' => data)
-      formatted = @formatter.format(tag, @time, record.dup)
-
-      r = record
-      r['foo'] = tag
-      assert_equal("#{Yajl.dump(r)}\n", formatted)
-    end
-
-    data('oj' => 'oj', 'yajl' => 'yajl')
-    def test_format_with_include_time(data)
-      @formatter.configure('include_time_key' => 'true', 'localtime' => '', 'json_parser' => data)
-      formatted = @formatter.format(tag, @time, record.dup)
-
-      r = record
-      r['time'] = time2str(@time, true)
-      assert_equal("#{Yajl.dump(r)}\n", formatted)
-    end
-
-    data('oj' => 'oj', 'yajl' => 'yajl')
-    def test_format_with_include_time_as_number(data)
-      @formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch', 'json_parser' => data)
-      formatted = @formatter.format(tag, @time, record.dup)
-
-      r = record
-      r['epoch'] = @time
-      assert_equal("#{Yajl.dump(r)}\n", formatted)
-    end
   end
 
   class MessagePackFormatterTest < ::Test::Unit::TestCase
@@ -183,33 +137,6 @@ module FormatterTest
 
       assert_equal(record.to_msgpack, formatted)
     end
-
-    def test_format_with_include_tag
-      @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo')
-      formatted = @formatter.format(tag, @time, record.dup)
-
-      r = record
-      r['foo'] = tag
-      assert_equal(r.to_msgpack, formatted)
-    end
-
-    def test_format_with_include_time
-      @formatter.configure('include_time_key' => 'true', 'localtime' => '')
-      formatted = @formatter.format(tag, @time, record.dup)
-
-      r = record
-      r['time'] = time2str(@time, true)
-      assert_equal(r.to_msgpack, formatted)
-    end
-
-    def test_format_with_include_time_as_number
-      @formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch')
-      formatted = @formatter.format(tag, @time, record.dup)
-
-      r = record
-      r['epoch'] = @time
-      assert_equal(r.to_msgpack, formatted)
-    end
   end
 
   class LabeledTSVFormatterTest < ::Test::Unit::TestCase
@@ -237,32 +164,17 @@ module FormatterTest
       @formatter.configure({})
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("message:awesome\n", formatted)
-    end
-
-    def test_format_with_tag
-      @formatter.configure('include_tag_key' => 'true')
-      formatted = @formatter.format(tag, @time, record)
-
-      assert_equal("message:awesome\ttag:tag\n", formatted)
-    end
-
-    def test_format_with_time
-      @formatter.configure('include_time_key' => 'true', 'time_format' => '%Y')
-      formatted = @formatter.format(tag, @time, record)
-
-      assert_equal("message:awesome\ttime:#{Time.now.year}\n", formatted)
+      assert_equal("message:awesome\tgreeting:hello\n", formatted)
     end
 
     def test_format_with_customized_delimiters
       @formatter.configure(
-        'include_tag_key' => 'true',
         'delimiter'       => ',',
         'label_delimiter' => '=',
       )
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("message=awesome,tag=tag\n", formatted)
+      assert_equal("message=awesome,greeting=hello\n", formatted)
     end
   end
 
@@ -273,11 +185,11 @@ module FormatterTest
       @formatter = TextFormatter::CsvFormatter.new
       @time = Engine.now
     end
-    
+
     def test_config_params
       assert_equal ',', @formatter.delimiter
       assert_equal true, @formatter.force_quotes
-      assert_equal [], @formatter.fields
+      assert_nil @formatter.fields
     end
 
     data(
@@ -286,8 +198,9 @@ module FormatterTest
       'pipe' => ['|', '|'])
     def test_config_params_with_customized_delimiters(data)
       expected, target = data
-      @formatter.configure('delimiter' => target)
+      @formatter.configure('delimiter' => target, 'fields' => 'a,b,c')
       assert_equal expected, @formatter.delimiter
+      assert_equal ['a', 'b', 'c'], @formatter.fields
     end
 
     def test_format
@@ -299,32 +212,6 @@ module FormatterTest
       assert_equal("\"awesome\",\"awesome2\"\n", formatted)
     end
 
-    def test_format_with_tag
-      @formatter.configure(
-        'fields' => 'tag,message,message2',
-        'include_tag_key' => 'true'
-      )
-      formatted = @formatter.format(tag, @time, {
-        'message' => 'awesome',
-        'message2' => 'awesome2'
-      })
-      assert_equal("\"tag\",\"awesome\",\"awesome2\"\n", formatted)
-    end
-
-    def test_format_with_time
-      @formatter.configure(
-        'fields' => 'time,message,message2',
-        'include_time_key' => 'true',
-        'time_format' => '%Y'
-      )
-      formatted = @formatter.format(tag, @time, {
-        'message' => 'awesome',
-        'message2' => 'awesome2'
-      })
-      assert_equal("\"#{Time.now.year}\",\"awesome\",\"awesome2\"\n",
-                   formatted)
-    end
-
     def test_format_with_customized_delimiters
       @formatter.configure(
         'fields' => 'message,message2',
@@ -429,48 +316,4 @@ module FormatterTest
       $LOAD_PATH.shift
     end
   end
-
-  class TimeConfigTest < ::Test::Unit::TestCase
-    include FormatterTest
-
-    def setup
-      @formatter = TextFormatter::LabeledTSVFormatter.new
-      @time      = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i
-    end
-
-    def format(conf)
-      @formatter.configure({'include_time_key' => true}.merge(conf))
-      formatted = @formatter.format("tag", @time, {})
-      # Drop the leading "time:" and the trailing "\n".
-      formatted[5..-2]
-    end
-
-    def test_none
-      with_timezone("UTC-01") do
-        # 'localtime' is true by default.
-        assert_equal("2014-09-27T01:00:00+01:00", format({}))
-      end
-    end
-
-    def test_utc
-      with_timezone("UTC-01") do
-        # 'utc' takes precedence over 'localtime'.
-        assert_equal("2014-09-27T00:00:00Z", format("utc" => true))
-      end
-    end
-
-    def test_timezone
-      with_timezone("UTC-01") do
-        # 'timezone' takes precedence over 'localtime'.
-        assert_equal("2014-09-27T02:00:00+02:00", format("timezone" => "+02"))
-      end
-    end
-
-    def test_utc_timezone
-      with_timezone("UTC-01") do
-        # 'timezone' takes precedence over 'utc'.
-        assert_equal("2014-09-27T09:00:00+09:00", format("utc" => true, "timezone" => "Asia/Tokyo"))
-      end
-    end
-  end
 end