fluentd 0.12.40 → 0.14.0

data/test/plugin/test_out_copy.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_copy'
 
 class CopyOutputTest < Test::Unit::TestCase
   class << self
@@ -107,7 +108,19 @@ class CopyOutputTest < Test::Unit::TestCase
 
   def create_event_test_driver(is_deep_copy = false)
     deep_copy_config = %[
-deep_copy true
+deep_copy #{is_deep_copy}
+    <store>
+      type test
+      name c0
+    </store>
+    <store>
+      type test
+      name c1
+    </store>
+    <store>
+      type test
+      name c2
+    </store>
 ]
 
     output1 = Fluent::Plugin.new_output('test')
@@ -130,7 +143,7 @@ deep_copy true
     outputs = [output1, output2]
 
     d = Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput)
-    d = d.configure(deep_copy_config) if is_deep_copy
+    d = d.configure(deep_copy_config)
     d.instance.instance_eval { @outputs = outputs }
     d
   end

data/test/plugin/test_out_exec.rb

@@ -1,11 +1,18 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_exec'
 require 'fileutils'
 
 class ExecOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.rm_rf(TMP_DIR, secure: true)
+    if File.exist?(TMP_DIR)
+      # ensure files are closed for Windows, on which deleted files
+      # are still visible from filesystem
+      GC.start(full_mark: true, immediate_sweep: true)
+      FileUtils.remove_entry_secure(TMP_DIR)
+    end
     FileUtils.mkdir_p(TMP_DIR)
   end
 
@@ -25,7 +32,7 @@ class ExecOutputTest < Test::Unit::TestCase
 
   def create_driver(conf = TSV_CONFIG)
     config = CONFIG + conf
-    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::ExecOutput).configure(config, true)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecOutput).configure(config)
   end
 
   def create_test_case
@@ -44,55 +51,98 @@ class ExecOutputTest < Test::Unit::TestCase
     assert_equal true, d.instance.localtime
   end
 
+  def test_configure_with_compat_buffer_parameters
+    conf = TSV_CONFIG + %[
+      buffer_type memory
+      time_slice_format %Y%m%d%H
+      num_threads 5
+      buffer_chunk_limit 50m
+      buffer_queue_limit 128
+      flush_at_shutdown yes
+    ]
+    d = create_driver(conf)
+    assert_equal 3600, d.instance.buffer_config.timekey
+    assert_equal 5, d.instance.buffer_config.flush_thread_count
+    assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size
+    assert_equal 128, d.instance.buffer.queue_length_limit
+    assert d.instance.buffer_config.flush_at_shutdown
+  end
+
   def test_format
     d = create_driver
     time, tests = create_test_case
 
-    tests.each { |test|
-      d.emit(test, time)
-    }
-
-    d.expect_format %[2011-01-02 13:14:15\ttest\tv1\n]
-    d.expect_format %[2011-01-02 13:14:15\ttest\tv2\n]
+    d.run(default_tag: 'test') do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
 
-    d.run
+    assert_equal %[2011-01-02 13:14:15\ttest\tv1\n], d.formatted[0]
+    assert_equal %[2011-01-02 13:14:15\ttest\tv2\n], d.formatted[1]
   end
 
   def test_format_json
     d = create_driver("format json")
     time, tests = create_test_case
 
-    tests.each { |test|
-      d.emit(test, time)
-      d.expect_format Yajl.dump(test) + "\n"
-    }
+    d.run(default_tag: 'test') do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
 
-    d.run
+    assert_equal Yajl.dump(tests[0]) + "\n", d.formatted[0]
+    assert_equal Yajl.dump(tests[1]) + "\n", d.formatted[1]
   end
 
   def test_format_msgpack
     d = create_driver("format msgpack")
     time, tests = create_test_case
 
-    tests.each { |test|
-      d.emit(test, time)
-      d.expect_format test.to_msgpack
-    }
+    d.run(default_tag: 'test') do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
+
+    assert_equal tests[0].to_msgpack, d.formatted[0]
+    assert_equal tests[1].to_msgpack, d.formatted[1]
+  end
+
+  def test_format_time
+    config = %[
+      keys "time,tag,k1"
+      tag_key "tag"
+      time_key "time"
+      time_format %Y-%m-%d %H:%M:%S.%3N
+    ]
+    d = create_driver(config)
 
-    d.run
+    time = event_time("2011-01-02 13:14:15.123")
+    tests = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
+
+    d.run(default_tag: 'test') do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
+
+    assert_equal %[2011-01-02 13:14:15.123\ttest\tv1\n], d.formatted[0]
+    assert_equal %[2011-01-02 13:14:15.123\ttest\tv2\n], d.formatted[1]
   end
 
   def test_write
     d = create_driver
     time, tests = create_test_case
 
-    tests.each { |test|
-      d.emit(test, time)
-    }
-
-    d.run
+    d.run(default_tag: 'test', flush: true) do
+      d.feed(time, tests[0])
+      d.feed(time, tests[1])
+    end
 
     expect_path = "#{TMP_DIR}/out"
+
+    waiting(10, plugin: d.instance) do
+      sleep(0.1) until File.exist?(expect_path)
+    end
+
     assert_equal true, File.exist?(expect_path)
 
     data = File.read(expect_path)

data/test/plugin/test_out_exec_filter.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_exec_filter'
 require 'fileutils'
 
 class ExecFilterOutputTest < Test::Unit::TestCase
@@ -20,12 +21,12 @@ class ExecFilterOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG, tag = 'test')
-    Fluent::Test::OutputTestDriver.new(Fluent::ExecFilterOutput, tag).configure(conf)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ExecFilterOutput, tag).configure(conf)
   end
 
   def sed_unbuffered_support?
     @sed_unbuffered_support ||= lambda {
-      system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >/dev/null 2>&1")
+      system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
       $?.success?
     }.call
   end
@@ -37,6 +38,8 @@ class ExecFilterOutputTest < Test::Unit::TestCase
   def test_configure
     d = create_driver
 
+    assert d.instance.instance_eval{ @overrides_format_stream }
+
     assert_equal ["time_in","tag","k1"], d.instance.in_keys
     assert_equal ["time_out","tag","k2"], d.instance.out_keys
     assert_equal "tag", d.instance.out_tag_key
@@ -69,8 +72,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
   def test_emit_1
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"k1"=>1}, time)
       d.emit({"k1"=>2}, time)
@@ -79,7 +83,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 2, emits.length
     assert_equal ["test", time, {"k2"=>"1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
     assert_equal ["test", time, {"k2"=>"2"}], emits[1]
+    assert_equal_event_time time, emits[1][1]
   end
 
   def test_emit_2
@@ -92,8 +98,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       num_children 3
     ]
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"k1"=>1}, time)
       d.emit({"k1"=>2}, time)
@@ -102,7 +109,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 2, emits.length
     assert_equal ["xxx", time, {"k2"=>"1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
     assert_equal ["xxx", time, {"k2"=>"2"}], emits[1]
+    assert_equal_event_time time, emits[1][1]
   end
 
   def test_emit_3
@@ -115,8 +124,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       num_children 3
     ]
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"val1"=>"sed-ed value foo"}, time)
       d.emit({"val1"=>"sed-ed value poo"}, time)
@@ -125,6 +135,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 1, emits.length
     assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
 
     d = create_driver %[
       command sed #{sed_unbuffered_option} -l -e s/foo/bar/
@@ -135,8 +146,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       num_children 3
     ]
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"val1"=>"sed-ed value foo"}, time)
       d.emit({"val1"=>"sed-ed value poo"}, time)
@@ -145,7 +157,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 2, emits.length
     assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
     assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], emits[1]
+    assert_equal_event_time time, emits[1][1]
   end
 
   def test_emit_4
@@ -160,8 +174,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       num_children 3
     ], 'input.test')
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 2
     d.run do
       d.emit({"val1"=>"sed-ed value foo"}, time)
       d.emit({"val1"=>"sed-ed value poo"}, time)
@@ -170,7 +185,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 2, emits.length
     assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
     assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], emits[1]
+    assert_equal_event_time time, emits[1][1]
   end
 
   def test_json_1
@@ -182,8 +199,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       tag_key tag
     ], 'input.test')
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
 
+    d.expected_emits_length = 1
     d.run do
       d.emit({"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]}, time+10)
     end
@@ -191,6 +209,54 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 1, emits.length
     assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
+  end
+
+  def test_json_with_float_time
+    d = create_driver(%[
+      command cat
+      in_keys message
+      out_format json
+      time_key time
+      tag_key tag
+    ], 'input.test')
+
+    float_time = Time.parse("2011-01-02 13:14:15").to_f
+    time = Fluent::EventTime.from_time(Time.at(float_time))
+
+    d.expected_emits_length = 1
+    d.run do
+      d.emit({"message"=>%[{"time":#{float_time},"tag":"t1","k1":"v1"}]}, time+10)
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
+  end
+
+  def test_json_with_time_format
+    d = create_driver(%[
+      command cat
+      in_keys message
+      out_format json
+      time_key time
+      time_format %d/%b/%Y %H:%M:%S.%N %z
+      tag_key tag
+    ], 'input.test')
+
+    time_str = "28/Feb/2013 12:00:00.123456789 +0900"
+    time = Fluent::EventTime.from_time(Time.strptime(time_str, "%d/%b/%Y %H:%M:%S.%N %z"))
+
+    d.expected_emits_length = 1
+    d.run do
+      d.emit({"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]}, time+10)
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
+    assert_equal_event_time time, emits[0][1]
   end
 end
 

data/test/plugin/test_out_file.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_file'
 require 'fileutils'
 require 'time'
 
@@ -62,7 +63,7 @@ class FileOutputTest < Test::Unit::TestCase
     d = create_driver(%[path #{TMP_DIR}/out_file_test])
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    with_timezone('Asia/Taipei') do
+    with_timezone(Fluent.windows? ? 'NST-8' : 'Asia/Taipei') do
       d.emit({"a"=>1}, time)
       d.expect_format %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}\n]
       d.run
@@ -121,7 +122,7 @@ class FileOutputTest < Test::Unit::TestCase
     # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
     # Following code from https://www.ruby-forum.com/topic/971591#979520
     result = ''
-    File.open(path) { |io|
+    File.open(path, "rb") { |io|
       loop do
         gzr = Zlib::GzipReader.new(io)
         result << gzr.read
@@ -150,6 +151,54 @@ class FileOutputTest < Test::Unit::TestCase
     check_gzipped_result(paths[0], %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
   end
 
+  class TestWithSystem < self
+    TMP_DIR_WITH_SYSTEM = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file_system#{ENV['TEST_ENV_NUMBER']}")
+    # 0750 interprets as "488". "488".to_i(8) # => 4. So, it makes wrong permission. Umm....
+    OVERRIDE_DIR_PERMISSION = 750
+    OVERRIDE_FILE_PERMISSION = 0620
+    CONFIG_WITH_SYSTEM = %[
+      path #{TMP_DIR_WITH_SYSTEM}/out_file_test
+      compress gz
+      utc
+      <system>
+        file_permission #{OVERRIDE_FILE_PERMISSION}
+        dir_permission #{OVERRIDE_DIR_PERMISSION}
+      </system>
+    ]
+
+    def setup
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+      FileUtils.rm_rf(TMP_DIR_WITH_SYSTEM)
+    end
+
+    def parse_system(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+    end
+
+    def test_write_with_system
+      system_conf = parse_system(CONFIG_WITH_SYSTEM)
+      sc = Fluent::SystemConfig.new(system_conf)
+      Fluent::Engine.init(sc)
+      d = create_driver CONFIG_WITH_SYSTEM
+
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      d.emit({"a"=>1}, time)
+      d.emit({"a"=>2}, time)
+
+      # FileOutput#write returns path
+      paths = d.run
+      expect_paths = ["#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz"]
+      assert_equal expect_paths, paths
+
+      check_gzipped_result(paths[0], %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
+      dir_mode = "%o" % File::stat(TMP_DIR_WITH_SYSTEM).mode
+      assert_equal(OVERRIDE_DIR_PERMISSION, dir_mode[-3, 3].to_i)
+      file_mode = "%o" % File::stat(paths[0]).mode
+      assert_equal(OVERRIDE_FILE_PERMISSION, file_mode[-3, 3].to_i)
+    end
+  end
+
   def test_write_with_format_json
     d = create_driver [CONFIG, 'format json', 'include_time_key true', 'time_as_epoch'].join("\n")
 
@@ -245,28 +294,33 @@ class FileOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_symlink
+    omit "Windows doesn't support symlink" if Fluent.windows?
     conf = CONFIG + %[
       symlink_path #{SYMLINK_PATH}
     ]
     symlink_path = "#{SYMLINK_PATH}"
 
-    Fluent::FileBuffer.clear_buffer_paths
     d = Fluent::Test::TestDriver.new(Fluent::FileOutput).configure(conf)
 
     begin
       d.instance.start
       10.times { sleep 0.05 }
+
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       es = Fluent::OneEventStream.new(time, {"a"=>1})
-      d.instance.emit('tag', es, Fluent::NullOutputChain.instance)
+      d.instance.emit_events('tag', es)
 
-      assert File.exists?(symlink_path)
+      assert File.exist?(symlink_path)
       assert File.symlink?(symlink_path)
 
-      d.instance.enqueue_buffer
+      es = Fluent::OneEventStream.new(event_time("2011-01-03 14:15:16 UTC"), {"a"=>2})
+      d.instance.emit_events('tag', es)
 
-      assert !File.exists?(symlink_path)
+      assert File.exist?(symlink_path)
       assert File.symlink?(symlink_path)
+
+      meta = d.instance.metadata('tag', event_time("2011-01-03 14:15:16 UTC"), {})
+      assert_equal d.instance.buffer.instance_eval{ @stage[meta].path }, File.readlink(symlink_path)
     ensure
       d.instance.shutdown
       FileUtils.rm_rf(symlink_path)