fluentd 0.12.43 → 0.14.0

data/test/plugin/test_in_tail.rb

@@ -1,15 +1,23 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_tail'
+require 'fluent/plugin/buffer'
+require 'fluent/system_config'
 require 'net/http'
 require 'flexmock/test_unit'
-require 'timecop'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
 
   def setup
     Fluent::Test.setup
-    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.rm_rf(TMP_DIR, secure: true)
+    if File.exist?(TMP_DIR)
+      # ensure files are closed for Windows, on which deleted files
+      # are still visible from filesystem
+      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
+      FileUtils.remove_entry_secure(TMP_DIR)
+    end
     FileUtils.mkdir_p(TMP_DIR)
   end
 
@@ -50,7 +58,6 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal 2, d.instance.rotate_wait
     assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
     assert_equal 1000, d.instance.read_lines_limit
-    assert_equal false, d.instance.ignore_repeated_permission_error
   end
 
   def test_configure_encoding
@@ -64,45 +71,20 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
-  def test_configure_from_encoding
-    # If only specified from_encoding raise ConfigError
-    assert_raise(Fluent::ConfigError) do
-      d = create_driver(SINGLE_LINE_CONFIG + 'from_encoding utf-8')
-    end
-
-    # valid setting
-    d = create_driver %[
-      format /(?<message>.*)/
-      read_from_head true
-      from_encoding utf-8
-      encoding utf-8
-    ]
-    assert_equal Encoding::UTF_8, d.instance.from_encoding
-
-    # invalid from_encoding
-    assert_raise(Fluent::ConfigError) do
-      d = create_driver %[
-        format /(?<message>.*)/
-        read_from_head true
-        from_encoding no-such-encoding
-        encoding utf-8
-      ]
-    end
-  end
-
   # TODO: Should using more better approach instead of sleep wait
 
   def test_emit
-    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
       f.puts "test1"
       f.puts "test2"
     }
 
     d = create_driver
+
     d.run do
       sleep 1
 
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
         f.puts "test3"
         f.puts "test4"
       }
@@ -113,35 +95,69 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal(true, emits.length > 0)
     assert_equal({"message" => "test3"}, emits[0][2])
     assert_equal({"message" => "test4"}, emits[1][2])
+    assert(emits[0][1].is_a?(Fluent::EventTime))
+    assert(emits[1][1].is_a?(Fluent::EventTime))
     assert_equal(1, d.emit_streams.size)
   end
 
-  def test_emit_with_emit_unmatched_lines_true
-    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+  class TestWithSystem < self
+    include Fluent::SystemConfig::Mixin
 
-    d = create_driver(%[
-      format /^(?<message>test.*)/
-      emit_unmatched_lines true
-    ])
-    d.run do
-      sleep 1
+    OVERRIDE_FILE_PERMISSION = 0620
+    CONFIG_SYSTEM = %[
+      <system>
+        file_permission #{OVERRIDE_FILE_PERMISSION}
+      </system>
+    ]
 
-      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
-        f.puts "test line 1"
-        f.puts "test line 2"
-        f.puts "bad line 1"
-        f.puts "test line 3"
-      }
+    def setup
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+      # Store default permission
+      @default_permission = system_config.instance_variable_get(:@file_permission)
+    end
 
-      sleep 1
+    def teardown
+      # Restore default permission
+      system_config.instance_variable_set(:@file_permission, @default_permission)
+    end
+
+    def parse_system(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
     end
 
-    events = d.emits
-    assert_equal(4, events.length)
-    assert_equal({"message" => "test line 1"}, events[0][2])
-    assert_equal({"message" => "test line 2"}, events[1][2])
-    assert_equal({"unmatched_line" => "bad line 1"}, events[2][2])
-    assert_equal({"message" => "test line 3"}, events[3][2])
+    def test_emit_with_system
+      system_conf = parse_system(CONFIG_SYSTEM)
+      sc = Fluent::SystemConfig.new(system_conf)
+      Fluent::Engine.init(sc)
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d = create_driver
+
+      d.run do
+        sleep 1
+
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+          f.puts "test3"
+          f.puts "test4"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      assert_equal({"message" => "test3"}, emits[0][2])
+      assert_equal({"message" => "test4"}, emits[1][2])
+      assert(emits[0][1].is_a?(Fluent::EventTime))
+      assert(emits[1][1].is_a?(Fluent::EventTime))
+      assert_equal(1, d.emit_streams.size)
+      pos = d.instance.instance_variable_get(:@pf_file)
+      mode = "%o" % File.stat(pos).mode
+      assert_equal OVERRIDE_FILE_PERMISSION, mode[-3, 3].to_i
+    end
   end
 
   data('1' => [1, 2], '10' => [10, 1])
@@ -153,7 +169,7 @@ class TailInputTest < Test::Unit::TestCase
     d.run do
       sleep 1
 
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
         f.puts msg
         f.puts msg
       }
@@ -168,7 +184,7 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def test_emit_with_read_from_head
-    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
       f.puts "test1"
       f.puts "test2"
     }
@@ -178,7 +194,7 @@ class TailInputTest < Test::Unit::TestCase
     d.run do
       sleep 1
 
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
         f.puts "test3"
         f.puts "test4"
       }
@@ -241,13 +257,13 @@ class TailInputTest < Test::Unit::TestCase
 
   def test_rotate_file_with_write_old
     emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
-      File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
       rotated_file.puts "test7"
       rotated_file.puts "test8"
       rotated_file.flush
 
       sleep 1
-      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
         f.puts "test5"
         f.puts "test6"
       }
@@ -275,7 +291,7 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def sub_test_rotate_file(config = nil)
-    file = File.open("#{TMP_DIR}/tail.txt", "w")
+    file = Fluent::FileWrapper.open("#{TMP_DIR}/tail.txt", "wb")
     file.puts "test1"
     file.puts "test2"
     file.flush
@@ -295,10 +311,10 @@ class TailInputTest < Test::Unit::TestCase
         sleep 1
       else
         sleep 1
-        File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+        File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
         sleep 1
 
-        File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+        File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
           f.puts "test5"
           f.puts "test6"
         }
@@ -312,69 +328,21 @@ class TailInputTest < Test::Unit::TestCase
 
     d.emits
   ensure
-    file.close
-  end
-
-  def test_truncate_file
-    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
-      f.puts "test1"
-      f.puts "test2"
-    }
-
-    d = create_driver
-    d.run do
-      sleep 1
-
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
-        f.puts "test3"
-        f.puts "test4"
-      }
-      sleep 1
-      File.truncate("#{TMP_DIR}/tail.txt", 6)
-      sleep 1
-    end
-
-    emits = d.emits
-    assert_equal(3, emits.length)
-    assert_equal({"message" => "test3"}, emits[0][2])
-    assert_equal({"message" => "test4"}, emits[1][2])
-    assert_equal({"message" => "test1"}, emits[2][2])
-  end
-
-  def test_move_truncate_move_back
-    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
-      f.puts "test1"
-      f.puts "test2"
-    }
-
-    d = create_driver
-    d.run do
-      sleep 1
-      FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
-      sleep 1
-      File.truncate("#{TMP_DIR}/tail2.txt", 6)
-      sleep 1
-      FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt")
-      sleep 1
-    end
-
-    emits = d.emits
-    assert_equal(1, emits.length)
-    assert_equal({"message" => "test1"}, emits[0][2])
+    file.close if file
   end
 
   def test_lf
-    File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
 
     d = create_driver
 
     d.run do
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
         f.print "test3"
       }
       sleep 1
 
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
         f.puts "test4"
       }
       sleep 1
@@ -386,14 +354,14 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def test_whitespace
-    File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
 
     d = create_driver
 
     d.run do
       sleep 1
 
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
         f.puts "    "		# 4 spaces
         f.puts "    4 spaces"
         f.puts "4 spaces    "
@@ -435,32 +403,10 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal(encoding, emits[0][2]['message'].encoding)
   end
 
-  def test_from_encoding
-    d = create_driver %[
-      format /(?<message>.*)/
-      read_from_head true
-      from_encoding cp932
-      encoding utf-8
-    ]
-
-    d.run do
-      sleep 1
-
-      File.open("#{TMP_DIR}/tail.txt", "w:cp932") {|f|
-        f.puts "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
-      }
-      sleep 1
-    end
-
-    emits = d.emits
-    assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message'])
-    assert_equal(Encoding::UTF_8, emits[0][2]['message'].encoding)
-  end
-
   # multiline mode test
 
   def test_multiline
-    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
 
     d = create_driver %[
       format multiline
@@ -468,7 +414,7 @@ class TailInputTest < Test::Unit::TestCase
       format_firstline /^[s]/
     ]
     d.run do
-      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
         f.puts "f test1"
         f.puts "s test2"
         f.puts "f test3"
@@ -496,44 +442,6 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal({"message1" => "test8"}, emits[3][2])
   end
 
-  def test_multiline_with_emit_unmatched_lines_true
-    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
-
-    d = create_driver %[
-      format multiline
-      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
-      format_firstline /^[s]/
-      emit_unmatched_lines true
-    ]
-    d.run do
-      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
-        f.puts "f test1"
-        f.puts "s test2"
-        f.puts "f test3"
-        f.puts "f test4"
-        f.puts "s test5"
-        f.puts "s test6"
-        f.puts "f test7"
-        f.puts "s test8"
-      }
-      sleep 1
-
-      events = d.emits
-      assert_equal(4, events.length)
-      assert_equal({"unmatched_line" => "f test1"}, events[0][2])
-      assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[1][2])
-      assert_equal({"message1" => "test5"}, events[2][2])
-      assert_equal({"message1" => "test6", "message2" => "test7"}, events[3][2])
-
-      sleep 3
-      assert_equal(4, d.emits.length)
-    end
-
-    emits = d.emits
-    assert_equal(5, emits.length)
-    assert_equal({"message1" => "test8"}, emits[4][2])
-  end
-
   def test_multiline_with_flush_interval
     File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
 
@@ -599,33 +507,8 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
-  def test_multiline_from_encoding_of_flushed_record
-    d = create_driver %[
-      format multiline
-      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
-      format_firstline /^[s]/
-      multiline_flush_interval 2s
-      read_from_head true
-      from_encoding cp932
-      encoding utf-8
-    ]
-
-    d.run do
-      sleep 1
-      File.open("#{TMP_DIR}/tail.txt", "w:cp932") { |f|
-        f.puts "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
-      }
-
-      sleep 4
-      emits = d.emits
-      assert_equal(1, emits.length)
-      assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message1'])
-      assert_equal(Encoding::UTF_8, emits[0][2]['message1'].encoding)
-    end
-  end
-
   def test_multiline_with_multiple_formats
-    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
 
     d = create_driver %[
       format multiline
@@ -635,7 +518,7 @@ class TailInputTest < Test::Unit::TestCase
       format_firstline /^[s]/
     ]
     d.run do
-      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
         f.puts "f test1"
         f.puts "s test2"
         f.puts "f test3"
@@ -658,7 +541,7 @@ class TailInputTest < Test::Unit::TestCase
 
   def test_multilinelog_with_multiple_paths
     files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
-    files.each { |file| File.open(file, "w") { |f| } }
+    files.each { |file| File.open(file, "wb") { |f| } }
 
     d = create_driver(%[
       path #{files[0]},#{files[1]}
@@ -669,7 +552,7 @@ class TailInputTest < Test::Unit::TestCase
     ], false)
     d.run do
       files.each do |file|
-        File.open(file, 'a') { |f|
+        File.open(file, 'ab') { |f|
           f.puts "f #{file} line should be ignored"
           f.puts "s test1"
           f.puts "f test2"
@@ -689,7 +572,7 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def test_multiline_without_firstline
-    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
 
     d = create_driver %[
       format multiline
@@ -698,7 +581,7 @@ class TailInputTest < Test::Unit::TestCase
       format3 /(?<var3>baz \\d)/
     ]
     d.run do
-      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
         f.puts "foo 1"
         f.puts "bar 1"
         f.puts "baz 1"
@@ -747,32 +630,13 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
   end
 
-  def test_log_file_without_extension
-    expected_files = [
-      'test/plugin/data/log/bar',
-      'test/plugin/data/log/foo/bar.log',
-      'test/plugin/data/log/foo/bar2',
-      'test/plugin/data/log/test.log'
-    ]
-
-    config = config_element("", "", {
-      "tag" => "tail",
-      "path" => "test/plugin/data/log/**/*",
-      "format" => "none",
-      "pos_file" => "#{TMP_DIR}/tail.pos"
-    })
-
-    plugin = create_driver(config, false).instance
-    assert_equal expected_files, plugin.expand_paths.sort
-  end
-
-  def test_refresh_watchers
+  def test_z_refresh_watchers
     plugin = create_driver(EX_CONFIG, false).instance
     sio = StringIO.new
     plugin.instance_eval do
       @pf = Fluent::NewTailInput::PositionFile.parse(sio)
       @loop = Coolio::Loop.new
-   end
+    end
 
     flexstub(Time) do |timeclass|
       timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
@@ -868,24 +732,12 @@ class TailInputTest < Test::Unit::TestCase
       engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
       plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
     end
-
-    config = %[
-      tag *
-      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
-      format none
-      read_from_head true
-    ]
-    plugin = create_driver(config, false).instance
-    flexstub(plugin.router) do |engineclass|
-      engineclass.should_receive(:emit_stream).with('foo.bar.log', any).once
-      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
-    end
   end
 
   # Ensure that no fatal exception is raised when a file is missing and that
   # files that do exist are still tailed as expected.
   def test_missing_file
-    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
       f.puts "test1"
       f.puts "test2"
     }
@@ -904,7 +756,7 @@ class TailInputTest < Test::Unit::TestCase
       d = create_driver(config, false)
       d.run do
         sleep 1
-        File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
           f.puts "test3"
           f.puts "test4"
         }
@@ -919,7 +771,7 @@ class TailInputTest < Test::Unit::TestCase
 
   sub_test_case 'emit error cases' do
     def test_emit_error_with_buffer_queue_limit_error
-      emits = execute_test(::Fluent::BufferQueueLimitError, "queue size exceeds limit")
+      emits = execute_test(Fluent::Plugin::Buffer::BufferOverflowError, "buffer space has too many data")
       assert_equal(10, emits.length)
       10.times { |i|
         assert_equal({"message" => "test#{i}"}, emits[i][2])
@@ -1073,43 +925,4 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal(files, [emits[2][2]["path"], emits[3][2]["path"]].sort)
     end
   end
-
-  def test_limit_recently_modified
-    now = Time.new(2010, 1, 2, 3, 4, 5)
-    FileUtils.touch("#{TMP_DIR}/tail_unwatch.txt", mtime: (now - 3601))
-    FileUtils.touch("#{TMP_DIR}/tail_watch1.txt", mtime: (now - 3600))
-    FileUtils.touch("#{TMP_DIR}/tail_watch2.txt", mtime: now)
-
-    config = config_element('', '', {
-      'tag' => 'tail',
-      'path' => "#{TMP_DIR}/*.txt",
-      'format' => 'none',
-      'limit_recently_modified' => '3600s'
-    })
-
-    expected_files = [
-      "#{TMP_DIR}/tail_watch1.txt",
-      "#{TMP_DIR}/tail_watch2.txt"
-    ]
-
-    Timecop.freeze(now) do
-      plugin = create_driver(config, false).instance
-      assert_equal expected_files, plugin.expand_paths.sort
-    end
-  end
-
-  def test_skip_refresh_on_startup
-    FileUtils.touch("#{TMP_DIR}/tail.txt")
-    config = config_element('', '', {
-                              'tag' => 'tail',
-                              'path' => "#{TMP_DIR}/*.txt",
-                              'format' => 'none',
-                              'refresh_interval' => 1,
-                              'skip_refresh_on_startup' => true
-                            })
-    d = create_driver(config, false)
-    d.run {
-      sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1
-    }
-  end
 end