fluentd 1.11.2-x64-mingw32 → 1.12.1-x64-mingw32

data/test/plugin/in_tail/test_position_file.rb

@@ -1,5 +1,6 @@
 require_relative '../../helper'
 require 'fluent/plugin/in_tail/position_file'
+require 'fluent/plugin/in_tail'
 
 require 'fileutils'
 require 'tempfile'
@@ -27,9 +28,15 @@ class IntailPositionFileTest < Test::Unit::TestCase
     f.seek(0)
   end
 
+  def follow_inodes_block
+    [true, false].each do |follow_inodes|
+      yield follow_inodes
+    end
+  end
+
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+    Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
     @file.seek(0)
     lines = @file.readlines
@@ -41,7 +48,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#try_compact' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -55,7 +62,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -64,7 +71,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change when the file is changed' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
 
       mock.proxy(pf).fetch_compacted_entries do |r|
         @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
@@ -78,12 +85,17 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 5, lines.size
     end
 
-    test 'update seek postion of remained position entry' do
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
-      pf['path1']
-      pf['path2']
-      pf['path3']
-      pf.unwatch('path1')
+    test 'update seek position of remained position entry' do
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
+      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
+      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
+      target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
+      pf[target_info1]
+      pf[target_info2]
+      pf[target_info3]
+
+      target_info1_2 = Fluent::Plugin::TailInput::TargetInfo.new('path1', 1234)
+      pf.unwatch(target_info1_2)
 
       pf.try_compact
 
@@ -93,8 +105,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
       assert_equal 2, lines.size
 
-      pf.unwatch('path2')
-      pf.unwatch('path3')
+      target_info2_2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', 1235)
+      target_info3_2 = Fluent::Plugin::TailInput::TargetInfo.new('path3', 1236)
+      pf.unwatch(target_info2_2)
+      pf.unwatch(target_info3_2)
       @file.seek(0)
       lines = @file.readlines
       assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
@@ -106,7 +120,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
       @file.seek(0)
       lines = @file.readlines
@@ -120,7 +134,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).load
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).load
 
       @file.seek(0)
       lines = @file.readlines
@@ -131,9 +145,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#[]' do
     test 'return entry' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
-      f = pf['valid_path']
+      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)
+      f = pf[valid_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 2, f.read_pos
       assert_equal 1, f.read_inode
@@ -142,7 +157,8 @@ class IntailPositionFileTest < Test::Unit::TestCase
       lines = @file.readlines
       assert_equal 2, lines.size
 
-      f = pf['nonexist_path']
+      nonexistent_target_info = Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)
+      f = pf[nonexistent_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 0, f.read_pos
       assert_equal 0, f.read_inode
@@ -155,19 +171,19 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change other value position if other entry try to write' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
 
-      f = pf['nonexist_path']
+      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf['valid_path'].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)].update(1, 2)
 
-      f = pf['nonexist_path']
+      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf['nonexist_path'].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)].update(1, 2)
       assert_equal 1, f.read_inode
       assert_equal 2, f.read_pos
     end
@@ -176,14 +192,18 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#unwatch' do
     test 'deletes entry by path' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
-      p1 = pf['valid_path']
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
+      inode1 = Fluent::FileWrapper.stat(@file).ino
+      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
+      p1 = pf[target_info1]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
 
-      pf.unwatch('valid_path')
+      pf.unwatch(target_info1)
       assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
 
-      p2 = pf['valid_path']
+      inode2 = Fluent::FileWrapper.stat(@file).ino
+      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
+      p2 = pf[target_info2]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
 
       assert_not_equal p1, p2

data/test/plugin/test_buffer.rb

@@ -543,6 +543,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
       assert_equal [@dm2,@dm3,m], @p.stage.keys
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 
@@ -675,6 +677,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm2,@dm3,m], @p.stage.keys
       assert_equal 1, @p.stage[m].append_count
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 

data/test/plugin/test_filter_stdout.rb

@@ -12,6 +12,11 @@ class StdoutFilterTest < Test::Unit::TestCase
     @old_tz = ENV["TZ"]
     ENV["TZ"] = "UTC"
     Timecop.freeze
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def teardown
@@ -106,7 +111,7 @@ class StdoutFilterTest < Test::Unit::TestCase
     def test_format_json
       d = create_driver(CONFIG + config_element("", "", { "format" => "json" }))
       out = capture_log(d) { filter(d, event_time, {'test' => 'test'}) }
-      assert_equal "{\"test\":\"test\"}\n", out
+      assert_equal "{\"test\":\"test\"}#{@default_newline}", out
     end
   end
 

data/test/plugin/test_formatter_hash.rb

@@ -19,11 +19,14 @@ class HashFormatterTest < ::Test::Unit::TestCase
     {'message' => 'awesome', 'greeting' => 'hello'}
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}\n!, formatted.encode(Encoding::UTF_8))
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}#{newline}!, formatted.encode(Encoding::UTF_8))
   end
 
   def test_format_without_newline

data/test/plugin/test_formatter_json.rb

@@ -7,6 +7,11 @@ class JsonFormatterTest < ::Test::Unit::TestCase
 
   def setup
     @time = event_time
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def create_driver(conf = "")
@@ -25,12 +30,17 @@ class JsonFormatterTest < ::Test::Unit::TestCase
     {:message => :awesome}
   end
 
-  data('oj' => 'oj', 'yajl' => 'yajl')
+  data('oj with LF' => ['oj', "lf", "\n"],
+       'oj with CRLF' => ['oj', "crlf", "\r\n"],
+       'yajl with LF' => ['yajl', "lf", "\n"],
+       'yajl with CRLF' => ['yajl', "crlf", "\r\n"]
+      )
   def test_format(data)
-    d = create_driver('json_parser' => data)
+    parser, newline_conf, newline = data
+    d = create_driver('json_parser' => parser, 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{JSON.generate(record)}\n", formatted)
+    assert_equal("#{JSON.generate(record)}#{newline}", formatted)
   end
 
   data('oj' => 'oj', 'yajl' => 'yajl')
@@ -46,6 +56,6 @@ class JsonFormatterTest < ::Test::Unit::TestCase
     d = create_driver('json_parser' => data)
     formatted = d.instance.format(tag, @time, symbolic_record)
 
-    assert_equal("#{JSON.generate(record)}\n", formatted)
+    assert_equal("#{JSON.generate(record)}#{@default_newline}", formatted)
   end
 end

data/test/plugin/test_formatter_ltsv.rb

@@ -36,11 +36,14 @@ class LabeledTSVFormatterTest < ::Test::Unit::TestCase
     assert_equal  false, d.instance.add_newline
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("message:awesome\tgreeting:hello\n", formatted)
+    assert_equal("message:awesome\tgreeting:hello#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -50,13 +53,18 @@ class LabeledTSVFormatterTest < ::Test::Unit::TestCase
     assert_equal("message:awesome\tgreeting:hello", formatted)
   end
 
-  def test_format_with_customized_delimiters
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_customized_delimiters(data)
+    newline_conf, newline = data
+
     d = create_driver(
       'delimiter'       => ',',
       'label_delimiter' => '=',
+      'newline' => newline_conf,
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("message=awesome,greeting=hello\n", formatted)
+    assert_equal("message=awesome,greeting=hello#{newline}", formatted)
   end
 end

data/test/plugin/test_formatter_out_file.rb

@@ -5,6 +5,11 @@ require 'fluent/plugin/formatter_out_file'
 class OutFileFormatterTest < ::Test::Unit::TestCase
   def setup
     @time = event_time
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def create_driver(conf = {})
@@ -48,48 +53,64 @@ class OutFileFormatterTest < ::Test::Unit::TestCase
       oldtz, ENV['TZ'] = ENV['TZ'], "UTC+07"
       d = create_driver(config_element('ROOT', '', {key => value}))
       tag = 'test'
-      assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}\n", d.instance.format(tag, time, record)
+      assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}#{@default_newline}", d.instance.format(tag, time, record)
     ensure
       ENV['TZ'] = oldtz
     end
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_time
-    d = create_driver('output_time' => 'false')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_time(data)
+    newline_conf, newline = data
+    d = create_driver('output_time' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{tag}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_tag
-    d = create_driver('output_tag' => 'false')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_tag(data)
+    newline_conf, newline = data
+    d = create_driver('output_tag' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
   def test_format_without_time_and_tag
-    d = create_driver('output_tag' => 'false', 'output_time' => 'false')
+    newline_conf, newline = data
+    d = create_driver('output_tag' => 'false', 'output_time' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format('tag', @time, record)
 
-    assert_equal("#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_time_and_tag_against_string_literal_configure
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_time_and_tag_against_string_literal_configure(data)
+    newline_conf, newline = data
     d = create_driver(%[
                         utc         true
                         output_tag  false
                         output_time false
+                        newline     #{newline_conf}
                       ])
     formatted = d.instance.format('tag', @time, record)
 
-    assert_equal("#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{Yajl.dump(record)}#{newline}", formatted)
   end
 end

data/test/plugin/test_formatter_single_value.rb

@@ -17,10 +17,13 @@ class SingleValueFormatterTest < ::Test::Unit::TestCase
     assert_equal "foobar", d.instance.message_key
   end
 
-  def test_format
-    d = create_driver
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver('newline' => newline_conf)
     formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
-    assert_equal("awesome\n", formatted)
+    assert_equal("awesome#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -29,10 +32,13 @@ class SingleValueFormatterTest < ::Test::Unit::TestCase
     assert_equal("awesome", formatted)
   end
 
-  def test_format_with_message_key
-    d = create_driver('message_key' => 'foobar')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_message_key(data)
+    newline_conf, newline = data
+    d = create_driver('message_key' => 'foobar', 'newline' => newline_conf)
     formatted = d.instance.format('tag', event_time, {'foobar' => 'foo'})
 
-    assert_equal("foo\n", formatted)
+    assert_equal("foo#{newline}", formatted)
   end
 end

data/test/plugin/test_formatter_tsv.rb

@@ -37,13 +37,17 @@ class TSVFormatterTest < ::Test::Unit::TestCase
     assert_equal false, d.instance.add_newline
   end
 
-  def test_format
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
     d = create_driver(
       'keys' => 'message,greeting',
+      'newline' => newline_conf
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("awesome\thello\n", formatted)
+    assert_equal("awesome\thello#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -56,13 +60,17 @@ class TSVFormatterTest < ::Test::Unit::TestCase
     assert_equal("awesome\thello", formatted)
   end
 
-  def test_format_with_customized_delimiters
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_customized_delimiters(data)
+    newline_conf, newline = data
     d = create_driver(
       'keys' => 'message,greeting',
       'delimiter' => ',',
+      'newline' => newline_conf,
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("awesome,hello\n", formatted)
+    assert_equal("awesome,hello#{newline}", formatted)
   end
 end