fluentd 0.12.43 → 0.14.0

data/test/plugin/test_out_buffered_null.rb

@@ -0,0 +1,79 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_buffered_null'
+
+class BufferedNullOutputTestCase < Test::Unit::TestCase
+  sub_test_case 'BufferedNullOutput' do
+    test 'default chunk limit size is 100' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedNullOutput).configure('')
+      assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
+      assert d.instance.buffer_config.flush_at_shutdown
+      assert_equal ['tag'], d.instance.buffer_config.chunk_keys
+      assert d.instance.chunk_key_tag
+      assert !d.instance.chunk_key_time
+      assert_equal [], d.instance.chunk_keys
+    end
+
+    test 'writes standard formattted chunks' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedNullOutput).configure('')
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 3, d.instance.emit_count
+      assert_equal 3, d.instance.emit_records
+    end
+
+    test 'check for chunk passed to #write' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedNullOutput).configure('')
+      data = []
+      d.instance.feed_proc = ->(chunk){ data << [chunk.unique_id, chunk.metadata.tag, chunk.read] }
+
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 1, data.size
+      _, tag, binary = data.first
+      events = []
+      Fluent::MessagePackFactory.unpacker.feed_each(binary){|obj| events << obj }
+      assert_equal 'test', tag
+      assert_equal [ [t, {"message" => "null null null"}], [t, {"message" => "null null"}], [t, {"message" => "null"}] ], events
+    end
+
+    test 'check for chunk passed to #try_write' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedNullOutput).configure('')
+      data = []
+      d.instance.feed_proc = ->(chunk){ data << [chunk.unique_id, chunk.metadata.tag, chunk.read] }
+      d.instance.delayed = true
+
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true, shutdown: false) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 1, data.size
+      chunk_id, tag, binary = data.first
+      events = []
+      Fluent::MessagePackFactory.unpacker.feed_each(binary){|obj| events << obj }
+      assert_equal 'test', tag
+      assert_equal [ [t, {"message" => "null null null"}], [t, {"message" => "null null"}], [t, {"message" => "null"}] ], events
+
+      assert_equal [chunk_id], d.instance.buffer.dequeued.keys
+
+      d.instance.commit_write(chunk_id)
+
+      assert_equal [], d.instance.buffer.dequeued.keys
+
+      d.instance_shutdown
+    end
+  end
+end

data/test/plugin/test_out_copy.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_copy'
 
 class CopyOutputTest < Test::Unit::TestCase
   class << self
@@ -107,7 +108,19 @@ class CopyOutputTest < Test::Unit::TestCase
 
   def create_event_test_driver(is_deep_copy = false)
     deep_copy_config = %[
-deep_copy true
+deep_copy #{is_deep_copy}
+    <store>
+      type test
+      name c0
+    </store>
+    <store>
+      type test
+      name c1
+    </store>
+    <store>
+      type test
+      name c2
+    </store>
 ]
 
     output1 = Fluent::Plugin.new_output('test')
@@ -130,7 +143,7 @@ deep_copy true
     outputs = [output1, output2]
 
     d = Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput)
-    d = d.configure(deep_copy_config) if is_deep_copy
+    d = d.configure(deep_copy_config)
     d.instance.instance_eval { @outputs = outputs }
     d
   end

data/test/plugin/test_out_exec.rb

@@ -1,11 +1,18 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_exec'
 require 'fileutils'
 
 class ExecOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.rm_rf(TMP_DIR, secure: true)
+    if File.exist?(TMP_DIR)
+      # ensure files are closed for Windows, on which deleted files
+      # are still visible from filesystem
+      GC.start(full_mark: true, immediate_sweep: true)
+      FileUtils.remove_entry_secure(TMP_DIR)
+    end
     FileUtils.mkdir_p(TMP_DIR)
   end
 
@@ -25,7 +32,7 @@ class ExecOutputTest < Test::Unit::TestCase
 
   def create_driver(conf = TSV_CONFIG)
     config = CONFIG + conf
-    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::ExecOutput).configure(config, true)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecOutput).configure(config)
   end
 
   def create_test_case
@@ -44,55 +51,98 @@ class ExecOutputTest < Test::Unit::TestCase
     assert_equal true, d.instance.localtime
   end
 
+  def test_configure_with_compat_buffer_parameters
+    conf = TSV_CONFIG + %[
+      buffer_type memory
+      time_slice_format %Y%m%d%H
+      num_threads 5
+      buffer_chunk_limit 50m
+      buffer_queue_limit 128
+      flush_at_shutdown yes
+    ]
+    d = create_driver(conf)
+    assert_equal 3600, d.instance.buffer_config.timekey
+    assert_equal 5, d.instance.buffer_config.flush_thread_count
+    assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size
+    assert_equal 128, d.instance.buffer.queue_length_limit
+    assert d.instance.buffer_config.flush_at_shutdown
+  end
+
   def test_format
     d = create_driver
     time, tests = create_test_case
 
-    tests.each { |test|
-      d.emit(test, time)
-    }
-
-    d.expect_format %[2011-01-02 13:14:15\ttest\tv1\n]
-    d.expect_format %[2011-01-02 13:14:15\ttest\tv2\n]
+    d.run(default_tag: 'test') do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
 
-    d.run
+    assert_equal %[2011-01-02 13:14:15\ttest\tv1\n], d.formatted[0]
+    assert_equal %[2011-01-02 13:14:15\ttest\tv2\n], d.formatted[1]
   end
 
   def test_format_json
     d = create_driver("format json")
     time, tests = create_test_case
 
-    tests.each { |test|
-      d.emit(test, time)
-      d.expect_format Yajl.dump(test) + "\n"
-    }
+    d.run(default_tag: 'test') do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
 
-    d.run
+    assert_equal Yajl.dump(tests[0]) + "\n", d.formatted[0]
+    assert_equal Yajl.dump(tests[1]) + "\n", d.formatted[1]
   end
 
   def test_format_msgpack
     d = create_driver("format msgpack")
     time, tests = create_test_case
 
-    tests.each { |test|
-      d.emit(test, time)
-      d.expect_format test.to_msgpack
-    }
+    d.run(default_tag: 'test') do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
+
+    assert_equal tests[0].to_msgpack, d.formatted[0]
+    assert_equal tests[1].to_msgpack, d.formatted[1]
+  end
+
+  def test_format_time
+    config = %[
+      keys "time,tag,k1"
+      tag_key "tag"
+      time_key "time"
+      time_format %Y-%m-%d %H:%M:%S.%3N
+    ]
+    d = create_driver(config)
 
-    d.run
+    time = event_time("2011-01-02 13:14:15.123")
+    tests = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
+
+    d.run(default_tag: 'test') do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
+
+    assert_equal %[2011-01-02 13:14:15.123\ttest\tv1\n], d.formatted[0]
+    assert_equal %[2011-01-02 13:14:15.123\ttest\tv2\n], d.formatted[1]
   end
 
   def test_write
     d = create_driver
     time, tests = create_test_case
 
-    tests.each { |test|
-      d.emit(test, time)
-    }
-
-    d.run
+    d.run(default_tag: 'test', flush: true) do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
 
     expect_path = "#{TMP_DIR}/out"
+
+    waiting(10, plugin: d.instance) do
+      sleep(0.1) until File.exist?(expect_path)
+    end
+
     assert_equal true, File.exist?(expect_path)
 
     data = File.read(expect_path)

data/test/plugin/test_out_exec_filter.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_exec_filter'
 require 'fileutils'
 
 class ExecFilterOutputTest < Test::Unit::TestCase
@@ -20,12 +21,12 @@ class ExecFilterOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG, tag = 'test')
-    Fluent::Test::OutputTestDriver.new(Fluent::ExecFilterOutput, tag).configure(conf)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ExecFilterOutput, tag).configure(conf)
   end
 
   def sed_unbuffered_support?
     @sed_unbuffered_support ||= lambda {
-      system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >/dev/null 2>&1")
+      system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
       $?.success?
     }.call
   end
@@ -37,6 +38,8 @@ class ExecFilterOutputTest < Test::Unit::TestCase
   def test_configure
     d = create_driver
 
+    assert d.instance.instance_eval{ @overrides_format_stream }
+
     assert_equal ["time_in","tag","k1"], d.instance.in_keys
     assert_equal ["time_out","tag","k2"], d.instance.out_keys
     assert_equal "tag", d.instance.out_tag_key
@@ -69,8 +72,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
   def test_emit_1
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"k1"=>1}, time)
       d.emit({"k1"=>2}, time)
@@ -79,7 +83,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 2, emits.length
     assert_equal ["test", time, {"k2"=>"1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
     assert_equal ["test", time, {"k2"=>"2"}], emits[1]
+    assert_equal_event_time time, emits[1][1]
   end
 
   def test_emit_2
@@ -92,8 +98,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       num_children 3
     ]
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"k1"=>1}, time)
       d.emit({"k1"=>2}, time)
@@ -102,7 +109,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 2, emits.length
     assert_equal ["xxx", time, {"k2"=>"1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
     assert_equal ["xxx", time, {"k2"=>"2"}], emits[1]
+    assert_equal_event_time time, emits[1][1]
   end
 
   def test_emit_3
@@ -115,8 +124,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       num_children 3
     ]
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"val1"=>"sed-ed value foo"}, time)
       d.emit({"val1"=>"sed-ed value poo"}, time)
@@ -125,6 +135,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 1, emits.length
     assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
 
     d = create_driver %[
       command sed #{sed_unbuffered_option} -l -e s/foo/bar/
@@ -135,8 +146,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       num_children 3
     ]
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"val1"=>"sed-ed value foo"}, time)
       d.emit({"val1"=>"sed-ed value poo"}, time)
@@ -145,7 +157,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 2, emits.length
     assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
     assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], emits[1]
+    assert_equal_event_time time, emits[1][1]
   end
 
   def test_emit_4
@@ -160,8 +174,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       num_children 3
     ], 'input.test')
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"val1"=>"sed-ed value foo"}, time)
       d.emit({"val1"=>"sed-ed value poo"}, time)
@@ -170,7 +185,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 2, emits.length
     assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
     assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], emits[1]
+    assert_equal_event_time time, emits[1][1]
   end
 
   def test_json_1
@@ -182,8 +199,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       tag_key tag
     ], 'input.test')
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 1
     d.run do
       d.emit({"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]}, time+10)
     end
@@ -191,6 +209,54 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 1, emits.length
     assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
+  end
+
+  def test_json_with_float_time
+    d = create_driver(%[
+      command cat
+      in_keys message
+      out_format json
+      time_key time
+      tag_key tag
+    ], 'input.test')
+
+    float_time = Time.parse("2011-01-02 13:14:15").to_f
+    time = Fluent::EventTime.from_time(Time.at(float_time))
+
+    d.expected_emits_length = 1
+    d.run do
+      d.emit({"message"=>%[{"time":#{float_time},"tag":"t1","k1":"v1"}]}, time+10)
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
+  end
+
+  def test_json_with_time_format
+    d = create_driver(%[
+      command cat
+      in_keys message
+      out_format json
+      time_key time
+      time_format %d/%b/%Y %H:%M:%S.%N %z
+      tag_key tag
+    ], 'input.test')
+
+    time_str = "28/Feb/2013 12:00:00.123456789 +0900"
+    time = Fluent::EventTime.from_time(Time.strptime(time_str, "%d/%b/%Y %H:%M:%S.%N %z"))
+
+    d.expected_emits_length = 1
+    d.run do
+      d.emit({"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]}, time+10)
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
   end
 end