fluentd 1.9.3 → 1.11.1

data/lib/fluent/system_config.rb

@@ -24,7 +24,7 @@ module Fluent
     SYSTEM_CONFIG_PARAMETERS = [
       :workers, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
-      :log_event_verbose,
+      :log_event_verbose, :ignore_repeated_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support
@@ -34,6 +34,7 @@ module Fluent
     config_param :root_dir,  :string, default: nil
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
+    config_param :ignore_repeated_log_interval, :time, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
     config_param :log_event_verbose,    :bool, default: nil

data/lib/fluent/test/filter_test.rb

@@ -30,12 +30,12 @@ module Fluent
       attr_reader :filtered
       attr_accessor :tag
 
-      def emit(record, time = Engine.now)
+      def emit(record, time = EventTime.now)
         emit_with_tag(@tag, record, time)
       end
       alias_method :filter, :emit
 
-      def emit_with_tag(tag, record, time = Engine.now)
+      def emit_with_tag(tag, record, time = EventTime.now)
         @events[tag] ||= MultiEventStream.new
         @events[tag].add(time, record)
       end

data/lib/fluent/test/output_test.rb

@@ -41,7 +41,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         es = OneEventStream.new(time, record)
         @instance.emit_events(@tag, es)
       end
@@ -62,7 +62,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end
@@ -110,7 +110,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.9.3'
+  VERSION = '1.11.1'
 
 end

data/test/command/test_fluentd.rb

@@ -16,6 +16,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     FileUtils.mkdir_p(TMP_DIR)
     @supervisor_pid = nil
     @worker_pids = []
+    ENV["TEST_RUBY_PATH"] = nil
   end
 
   def process_exist?(pid)
@@ -84,9 +85,22 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     null_stream.close rescue nil
   end
 
+  def eager_read(io)
+    buf = +''
+
+    loop do
+      b = io.read_nonblock(1024, nil, exception: false)
+      if b == :wait_readable || b.nil?
+        return buf
+      end
+      buf << b
+    end
+  end
+
   def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
     matched = false
-    assert_error_msg = "matched correctly"
+    matched_wrongly = false
+    assert_error_msg = ""
     stdio_buf = ""
     begin
       execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
@@ -97,7 +111,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               next unless readables
               break if readables.first.eof?
 
-              buf = readables.first.readpartial(1024)
+              buf = eager_read(readables.first)
               # puts buf
               stdio_buf << buf
               lines = stdio_buf.split("\n")
@@ -116,13 +130,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
         end
       end
     rescue Timeout::Error
-      assert_error_msg = "execution timeout with command out:\n" + stdio_buf
+      assert_error_msg = "execution timeout"
     rescue => e
-      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
+      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
+    else
+      assert_error_msg = "log doesn't match" unless matched
     end
-    assert matched, assert_error_msg
 
-    unless patterns_not_match.empty?
+    if patterns_not_match.empty?
+      assert_error_msg = build_message(assert_error_msg,
+                                       "<?>\nwas expected to include:\n<?>",
+                                       stdio_buf, pattern_list)
+    else
       lines = stdio_buf.split("\n")
       patterns_not_match.each do |ptn|
         matched_wrongly = if ptn.is_a? Regexp
@@ -130,9 +149,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
                           else
                             lines.any?{|line| line.include?(ptn) }
                           end
-        assert_false matched_wrongly, "pattern exists in logs wrongly:\n" + stdio_buf
+        if matched_wrongly
+          assert_error_msg << "\n" unless assert_error_msg.empty?
+          assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
+        end
       end
+      assert_error_msg = build_message(assert_error_msg,
+                                       "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
+                                       stdio_buf, pattern_list, patterns_not_match)
     end
+
+    assert matched && !matched_wrongly, assert_error_msg
   end
 
   def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
@@ -150,7 +177,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               next unless readables
               next if readables.first.eof?
 
-              stdio_buf << readables.first.readpartial(1024)
+              stdio_buf << eager_read(readables.first)
               lines = stdio_buf.split("\n")
               if lines.any?{|line| line.include?("fluentd worker is now running") }
                 running = true
@@ -830,8 +857,7 @@ CONF
       '-external-encoding' => '--external-encoding=utf-8',
       '-internal-encoding' => '--internal-encoding=utf-8',
     )
-    test "-E option is set to RUBYOPT3" do |opt|
-      omit "hard to run correctly on Windows. Need to debug." if Fluent.windows?
+    test "-E option is set to RUBYOPT" do |opt|
       conf = <<CONF
 <source>
   @type dummy
@@ -842,6 +868,7 @@ CONF
 </match>
 CONF
       conf_path = create_conf_file('rubyopt_test.conf', conf)
+      opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
       assert_log_matches(
         create_cmdline(conf_path),
         *opt.split(' '),
@@ -850,7 +877,7 @@ CONF
       )
     end
 
-        test "without RUBYOPT" do
+    test "without RUBYOPT" do
       conf = <<CONF
 <source>
   @type dummy
@@ -865,7 +892,7 @@ CONF
     end
 
     test 'invalid values are set to RUBYOPT' do
-      omit "hard to run correctly on Windows. Need to debug." if Fluent.windows?
+      omit "hard to run correctly because RUBYOPT=-r/path/to/bundler/setup is required on Windows while this test set invalid RUBYOPT" if Fluent.windows?
       conf = <<CONF
 <source>
   @type dummy
@@ -883,6 +910,38 @@ CONF
       )
     end
 
+    # https://github.com/fluent/fluentd/issues/2915
+    test "ruby path contains spaces" do
+      conf = <<CONF
+<source>
+  @type dummy
+  tag dummy
+</source>
+<match>
+  @type null
+</match>
+CONF
+      ruby_path = ServerEngine.ruby_bin_path
+      tmp_ruby_path = File.join(TMP_DIR, "ruby with spaces")
+      if Fluent.windows?
+        tmp_ruby_path << ".bat"
+        File.open(tmp_ruby_path, "w") do |file|
+          file.write "#{ruby_path} %*"
+        end
+      else
+        FileUtils.ln_sf(ruby_path, tmp_ruby_path)
+      end
+      ENV["TEST_RUBY_PATH"] = tmp_ruby_path
+      cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
+      conf_path = create_conf_file('space_mixed_ruby_path_test.conf', conf)
+      args = ["bundle", "exec", tmp_ruby_path, cmd_path, "-c", conf_path]
+      assert_log_matches(
+        args,
+        'spawn command to main:',
+        '-Eascii-8bit:ascii-8bit'
+      )
+    end
+
     test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
       conf = <<CONF
 <system>

data/test/config/test_system_config.rb

@@ -73,6 +73,7 @@ module Fluent::Config
       assert_nil(sc.root_dir)
       assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
+      assert_nil(sc.ignore_repeated_log_interval)
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
@@ -86,6 +87,7 @@ module Fluent::Config
       'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
       'log_level' => ['log_level', 'error'],
       'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
+      'ignore_repeated_log_interval' => ['ignore_repeated_log_interval', 10],
       'log_event_verbose' => ['log_event_verbose', true],
       'suppress_config_dump' => ['suppress_config_dump', true],
       'without_source' => ['without_source', true],

data/test/helper.rb

@@ -49,8 +49,8 @@ require 'fluent/plugin_helper'
 require 'fluent/msgpack_factory'
 require 'fluent/time'
 require 'serverengine'
-require 'helpers/fuzzy_assert'
-require 'helpers/process_extenstion'
+require_relative 'helpers/fuzzy_assert'
+require_relative 'helpers/process_extenstion'
 
 module Fluent
   module Plugin

data/test/plugin/in_tail/test_fifo.rb

@@ -0,0 +1,121 @@
+require_relative '../../helper'
+
+require 'fluent/plugin/in_tail'
+
+class IntailFIFO < Test::Unit::TestCase
+  sub_test_case '#read_line' do
+    test 'returns lines spliting per `\n`' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    test 'concant line when line is separated' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3 + 'test').force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      fifo << "2\n"
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n", "test2\n"], lines
+    end
+
+    test 'returns lines which convert encoding' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::UTF_8)
+      text = ("test\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::UTF_8, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    test 'reads lines as from_encoding' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::UTF_8)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    sub_test_case 'when it includes multi byte chars' do
+      test 'handles it as ascii_8bit' do
+        fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+        text = ("てすと\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+        fifo << text
+        lines = []
+        fifo.read_lines(lines)
+        assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+        assert_equal ["てすと\n", "てすと\n", "てすと\n"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines
+      end
+
+      test 'replaces character with ? when convert error happens' do
+        fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, Encoding::ASCII_8BIT)
+        text = ("てすと\n" * 3).force_encoding(Encoding::UTF_8)
+        fifo << text
+        lines = []
+        fifo.read_lines(lines)
+        assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+        assert_equal ["???\n", "???\n", "???\n"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines
+      end
+    end
+
+    test 'reutrns nothing when buffer is empty' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal [], lines
+
+      text = "test\n" * 3
+      fifo << text
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal [], lines
+    end
+  end
+
+  sub_test_case '#<<' do
+    test 'does not make any change about encoding to an argument' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::UTF_8)
+
+      assert_equal Encoding::UTF_8, text.encoding
+      fifo << text
+      assert_equal Encoding::UTF_8, text.encoding
+    end
+  end
+
+  sub_test_case '#bytesize' do
+    test 'reutrns buffer size' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = "test\n" * 3 + 'test'
+      fifo << text
+
+      assert_equal text.bytesize, fifo.bytesize
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      assert_equal 'test'.bytesize, fifo.bytesize
+      fifo << "2\n"
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n", "test2\n"], lines
+
+      assert_equal 0, fifo.bytesize
+    end
+  end
+end

data/test/plugin/in_tail/test_io_handler.rb

@@ -0,0 +1,132 @@
+require_relative '../../helper'
+
+require 'fluent/plugin/in_tail'
+require 'tempfile'
+
+class IntailIOHandlerTest < Test::Unit::TestCase
+  setup do
+    @file = Tempfile.new('intail_io_handler').binmode
+  end
+
+  teardown do
+    @file.close rescue nil
+    @file.unlink rescue nil
+  end
+
+  test '#on_notify load file content and passed it to receive_lines method' do
+    text = "this line is test\ntest line is test\n"
+    @file.write(text)
+    @file.close
+
+    watcher = 'watcher'
+
+    update_pos = 0
+
+    stub(watcher).pe do
+      pe = 'position_file'
+      stub(pe).read_pos { 0 }
+      stub(pe).update_pos { |val| update_pos = val }
+      pe
+    end
+
+    returned_lines = ''
+    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: false) do |lines, _watcher|
+      returned_lines << lines.join
+      true
+    end
+
+    r.on_notify
+    assert_equal text.bytesize, update_pos
+    assert_equal text, returned_lines
+
+    r.on_notify
+
+    assert_equal text.bytesize, update_pos
+    assert_equal text, returned_lines
+  end
+
+  sub_test_case 'when open_on_every_update is true and read_pos returns always 0' do
+    test 'open new IO and change pos to 0 and read it' do
+      text = "this line is test\ntest line is test\n"
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = 'watcher'
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = ''
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: true) do |lines, _watcher|
+        returned_lines << lines.join
+        true
+      end
+
+      r.on_notify
+      assert_equal text.bytesize, update_pos
+      assert_equal text, returned_lines
+
+      r.on_notify
+      assert_equal text * 2, returned_lines
+    end
+  end
+
+  sub_test_case 'when limit is 5' do
+    test 'call receive_lines once when short line(less than 8192)' do
+      text = "line\n" * 8
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = 'watcher'
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = []
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
+        returned_lines << lines.dup
+        true
+      end
+
+      r.on_notify
+      assert_equal 8, returned_lines[0].size
+    end
+
+    test 'call receive_lines some times when long line(more than 8192)' do
+      t = 'line' * (8192 / 8)
+      text = "#{t}\n" * 8
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = 'watcher'
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = []
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
+        returned_lines << lines.dup
+        true
+      end
+
+      r.on_notify
+      assert_equal 5, returned_lines[0].size
+      assert_equal 3, returned_lines[1].size
+    end
+  end
+end