fluentd 0.14.1 → 0.14.2

data/test/plugin/test_formatter_msgpack.rb

@@ -0,0 +1,28 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_msgpack'
+
+class MessagePackFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::MessagePackFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome'}
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(record.to_msgpack, formatted)
+  end
+end

data/test/plugin/test_formatter_out_file.rb

@@ -0,0 +1,95 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_out_file'
+
+class OutFileFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = {})
+    d = Fluent::Test::Driver::Formatter.new(Fluent::Plugin::OutFileFormatter)
+    case conf
+    when Fluent::Config::Element
+      d.configure(conf)
+    when Hash
+      d.configure({'utc' => true}.merge(conf))
+    else
+      d.configure(conf)
+    end
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome'}
+  end
+
+  data('both true' => 'true', 'both false' => 'false')
+  def test_configured_with_both_of_utc_and_localtime(value)
+    assert_raise(Fluent::ConfigError.new("both of utc and localtime are specified, use only one of them")) do
+      create_driver({'utc' => value, 'localtime' => value})
+    end
+  end
+
+  time_i = Time.parse("2016-07-26 21:08:30 -0700").to_i
+  data(
+    'configured for localtime by localtime' => ['localtime', 'true',  time_i, "2016-07-26T21:08:30-07:00"],
+    'configured for localtime by utc'       => ['utc',       'false', time_i, "2016-07-26T21:08:30-07:00"],
+    'configured for utc by localtime'       => ['localtime', 'false', time_i, "2016-07-27T04:08:30Z"],
+    'configured for utc by utc'             => ['utc',       'true',  time_i, "2016-07-27T04:08:30Z"],
+  )
+  def test_configured_with_utc_or_localtime(data)
+    key, value, time_i, expected = data
+    time = Time.at(time_i)
+    begin
+      oldtz, ENV['TZ'] = ENV['TZ'], "UTC+07"
+      d = create_driver(config_element('ROOT', '', {key => value}))
+      tag = 'test'
+      assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}\n", d.instance.format(tag, time, record)
+    ensure
+      ENV['TZ'] = oldtz
+    end
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time
+    d = create_driver('output_time' => 'false')
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_tag
+    d = create_driver('output_tag' => 'false')
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time_and_tag
+    d = create_driver('output_tag' => 'false', 'output_time' => 'false')
+    formatted = d.instance.format('tag', @time, record)
+
+    assert_equal("#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time_and_tag_against_string_literal_configure
+    d = create_driver(%[
+                        utc         true
+                        output_tag  false
+                        output_time false
+                      ])
+    formatted = d.instance.format('tag', @time, record)
+
+    assert_equal("#{Yajl.dump(record)}\n", formatted)
+  end
+end

data/test/plugin/test_formatter_single_value.rb

@@ -0,0 +1,38 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_single_value'
+
+class SingleValueFormatterTest < ::Test::Unit::TestCase
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::SingleValueFormatter).configure(conf)
+  end
+
+  def test_config_params
+    d = create_driver
+    assert_equal "message", d.instance.message_key
+  end
+
+  def test_config_params_message_key
+    d = create_driver('message_key' => 'foobar')
+    assert_equal "foobar", d.instance.message_key
+  end
+
+  def test_format
+    d = create_driver
+    formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
+    assert_equal("awesome\n", formatted)
+  end
+
+  def test_format_without_newline
+    d = create_driver('add_newline' => 'false')
+    formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
+    assert_equal("awesome", formatted)
+  end
+
+  def test_format_with_message_key
+    d = create_driver('message_key' => 'foobar')
+    formatted = d.instance.format('tag', event_time, {'foobar' => 'foo'})
+
+    assert_equal("foo\n", formatted)
+  end
+end

data/test/plugin/test_in_dummy.rb

@@ -1,6 +1,7 @@
 require_relative '../helper'
 require 'fluent/test/driver/input'
 require 'fluent/plugin/in_dummy'
+require 'fileutils'
 
 class DummyTest < Test::Unit::TestCase
   def setup
@@ -90,4 +91,98 @@ class DummyTest < Test::Unit::TestCase
       end
     end
   end
+
+  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
+  FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
+
+  sub_test_case "doesn't suspend internal counters in default" do
+    config1 = {
+      'tag' => 'dummy',
+      'rate' => '2',
+      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'auto_increment_key' => 'id',
+      'suspend' => false,
+    }
+    conf1 = config_element('ROOT', '', config1, [])
+    test "value of auto increment key is not suspended after stop-and-start" do
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+
+      d1 = create_driver(conf1)
+      d1.run(timeout: 0.5) do
+        d1.instance.emit(4)
+      end
+
+      first_id1 = d1.events.first[2]['id']
+      assert_equal 0, first_id1
+
+      last_id1 = d1.events.last[2]['id']
+      assert { last_id1 > 0 }
+
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+
+      d2 = create_driver(conf1)
+      d2.run(timeout: 0.5) do
+        d2.instance.emit(4)
+      end
+
+      first_id2 = d2.events.first[2]['id']
+      assert_equal 0, first_id2
+
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+    end
+  end
+
+  sub_test_case "suspend internal counters if suspend is true" do
+    setup do
+      FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
+      FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+      FileUtils.chmod_R(0755, File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+    end
+
+    config2 = {
+      '@id' => 'test-02',
+      'tag' => 'dummy',
+      'rate' => '2',
+      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'auto_increment_key' => 'id',
+      'suspend' => true,
+    }
+    conf2 = config_element('ROOT', '', config2, [
+              config_element(
+                'storage', '',
+                {'@type' => 'local',
+                 '@id' => 'test-02',
+                 'path' => File.join(TEST_PLUGIN_STORAGE_PATH,
+                                     'json', 'test-02.json'),
+                 'persistent' => true,
+                })
+            ])
+    test "value of auto increment key is suspended after stop-and-start" do
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d1 = create_driver(conf2)
+      d1.run(timeout: 0.5) do
+        d1.instance.emit(4)
+      end
+
+      first_id1 = d1.events.first[2]['id']
+      assert_equal 0, first_id1
+
+      last_id1 = d1.events.last[2]['id']
+      assert { last_id1 > 0 }
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d2 = create_driver(conf2)
+      d2.run(timeout: 0.5) do
+        d2.instance.emit(4)
+      end
+      d2.events
+
+      first_id2 = d2.events.first[2]['id']
+      assert_equal last_id1 + 1, first_id2
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+    end
+  end
 end

data/test/plugin/test_in_exec.rb

@@ -1,17 +1,17 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_exec'
 require 'net/http'
 
 class ExecInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @test_time = Fluent::EventTime.parse("2011-01-02 13:14:15")
+    @test_time = event_time("2011-01-02 13:14:15")
     @script = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
   end
 
   def create_driver(conf = tsv_config)
-    Fluent::Test::InputTestDriver.new(Fluent::ExecInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ExecInput).configure(conf)
   end
 
   def tsv_config
@@ -86,52 +86,48 @@ class ExecInputTest < Test::Unit::TestCase
   def test_emit
     d = create_driver
 
-    d.run do
-      sleep 2
-    end
+    d.run(expect_emits: 2)
 
-    emits = d.emits
-    assert_equal true, emits.length > 0
-    assert_equal ["tag1", @test_time, {"k1"=>"ok"}], emits[0]
-    assert_equal_event_time(@test_time, emits[0][1])
+    assert_equal true, d.events.length > 0
+    d.events.each_with_index {|event, i|
+      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
+      assert_equal_event_time(@test_time, event[1])
+    }
   end
 
   def test_emit_json
     d = create_driver json_config
 
-    d.run do
-      sleep 2
-    end
+    d.run(expect_emits: 2)
 
-    emits = d.emits
-    assert_equal true, emits.length > 0
-    assert_equal ["tag1", @test_time, {"k1"=>"ok"}], emits[0]
-    assert_equal_event_time(@test_time, emits[0][1])
+    assert_equal true, d.events.length > 0
+    d.events.each_with_index {|event, i|
+      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
+      assert_equal_event_time(@test_time, event[1])
+    }
   end
 
   def test_emit_msgpack
     d = create_driver msgpack_config
 
-    d.run do
-      sleep 2
-    end
+    d.run(expect_emits: 2)
 
-    emits = d.emits
-    assert_equal true, emits.length > 0
-    assert_equal ["tag1", @test_time, {"k1"=>"ok"}], emits[0]
-    assert_equal_event_time(@test_time, emits[0][1])
+    assert_equal true, d.events.length > 0
+    d.events.each_with_index {|event, i|
+      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
+      assert_equal_event_time(@test_time, event[1])
+    }
   end
 
   def test_emit_regexp
     d = create_driver regexp_config
 
-    d.run do
-      sleep 2
-    end
+    d.run(expect_emits: 2)
 
-    emits = d.emits
-    assert_equal true, emits.length > 0
-    assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], emits[0]
-    assert_equal_event_time(@test_time, emits[0][1])
+    assert_equal true, d.events.length > 0
+    d.events.each_with_index {|event, i|
+      assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], event
+      assert_equal_event_time(@test_time, event[1])
+    }
   end
 end

data/test/plugin/test_in_forward.rb

@@ -242,6 +242,30 @@ class ForwardInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_set_size_to_option
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+
+    entries = ''
+    events.each {|_tag, _time, record|
+      [_time, record].to_msgpack(entries)
+    }
+
+    chunk = ["tag1", entries, { 'size' => events.length }].to_msgpack
+
+    d.run do
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+        option = d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        assert_equal option['size'], events.length
+      end
+    end
+  end
+
   def test_send_large_chunk_warning
     d = create_driver(CONFIG + %[
       chunk_size_warn_limit 16M

data/test/plugin/test_in_gc_stat.rb

@@ -27,13 +27,13 @@ class GCStatInputTest < Test::Unit::TestCase
     stub(GC).stat { stat }
 
     d = create_driver
-    d.run do
-      sleep 2
-    end
+    d.run(expect_emits: 2)
 
     events = d.events
     assert(events.length > 0)
-    assert_equal(stat, events[0][2])
-    assert(events[0][1].is_a?(Fluent::EventTime))
+    events.each_index {|i|
+      assert_equal(stat, events[i][2])
+      assert(events[i][1].is_a?(Fluent::EventTime))
+    }
   end
 end

data/test/plugin/test_in_object_space.rb

@@ -1,5 +1,5 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_object_space'
 
 require 'timeout'
@@ -33,7 +33,7 @@ class ObjectSpaceInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=TESTCONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::ObjectSpaceInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ObjectSpaceInput).configure(conf)
   end
 
   def test_configure
@@ -48,11 +48,11 @@ class ObjectSpaceInputTest < Test::Unit::TestCase
 
     d.run do
       waiting(10, d.instance) do
-        sleep 0.5 until d.emit_streams.size > 3
+        sleep 0.5 until d.events.size > 3
       end
     end
 
-    emits = d.emits
+    emits = d.events
     assert{ emits.length > 0 }
 
     emits.each { |tag, time, record|