fluentd 1.13.3 → 1.16.5

data/test/test_log.rb

@@ -4,13 +4,17 @@ require 'fluent/engine'
 require 'fluent/log'
 require 'timecop'
 require 'logger'
+require 'securerandom'
+require 'pathname'
 
 class LogTest < Test::Unit::TestCase
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/log/#{ENV['TEST_ENV_NUMBER']}")
+  def tmp_dir
+    File.join(File.dirname(__FILE__), "tmp", "log", "#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
+  end
 
   def setup
-    FileUtils.rm_rf(TMP_DIR)
-    FileUtils.mkdir_p(TMP_DIR)
+    @tmp_dir = tmp_dir
+    FileUtils.mkdir_p(@tmp_dir)
     @log_device = Fluent::Test::DummyLogDevice.new
     @timestamp = Time.parse("2016-04-21 02:58:41 +0000")
     @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
@@ -21,6 +25,21 @@ class LogTest < Test::Unit::TestCase
     @log_device.reset
     Timecop.return
     Thread.current[:last_repeated_stacktrace] = nil
+    begin
+      FileUtils.rm_rf(@tmp_dir)
+    rescue Errno::EACCES
+      # It may occur on Windows because of delete pending state due to delayed GC.
+      # Ruby 3.2 or later doesn't ignore Errno::EACCES:
+      # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
+    end
+  end
+
+  def test_per_process_path
+    path = Fluent::Log.per_process_path("C:/tmp/test.log", :supervisor, 0)
+    assert_equal(path, "C:/tmp/test-supervisor-0.log")
+
+    path = Fluent::Log.per_process_path("C:/tmp/test.log", :worker, 1)
+    assert_equal(path, "C:/tmp/test-1.log")
   end
 
   sub_test_case "log level" do
@@ -453,6 +472,43 @@ class LogTest < Test::Unit::TestCase
       ]
       assert_equal(expected, log.out.logs)
     end
+
+    def test_reject_on_max_size
+      ignore_same_log_interval = 10
+
+      logger = Fluent::Log.new(
+        ServerEngine::DaemonLogger.new(@log_device, log_level: ServerEngine::DaemonLogger::INFO),
+        ignore_same_log_interval: ignore_same_log_interval,
+      )
+
+      # Output unique log every second.
+      Fluent::Log::IGNORE_SAME_LOG_MAX_CACHE_SIZE.times do |i|
+        logger.info "Test #{i}"
+        Timecop.freeze(@timestamp + i)
+      end
+      logger.info "Over max size!"
+
+      # The newest cache and the latest caches in `ignore_same_log_interval` should exist.
+      assert { Thread.current[:last_same_log].size == ignore_same_log_interval + 1 }
+    end
+
+    def test_clear_on_max_size
+      ignore_same_log_interval = 10
+
+      logger = Fluent::Log.new(
+        ServerEngine::DaemonLogger.new(@log_device, log_level: ServerEngine::DaemonLogger::INFO),
+        ignore_same_log_interval: ignore_same_log_interval,
+      )
+
+      # Output unique log at the same time.
+      Fluent::Log::IGNORE_SAME_LOG_MAX_CACHE_SIZE.times do |i|
+        logger.info "Test #{i}"
+      end
+      logger.info "Over max size!"
+
+      # Can't reject old logs, so all cache should be cleared and only the newest should exist.
+      assert { Thread.current[:last_same_log].size == 1 }
+    end
   end
 
   def test_dup
@@ -560,7 +616,7 @@ class LogTest < Test::Unit::TestCase
       Timecop.freeze(@timestamp)
 
       rotate_age, rotate_size, travel_term = expected
-      path = "#{TMP_DIR}/log-dev-io-#{rotate_size}-#{rotate_age}"
+      path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
 
       logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
       logger = ServerEngine::DaemonLogger.new(logdev)
@@ -583,45 +639,71 @@ class LogTest < Test::Unit::TestCase
 
   def test_log_rotates_specified_size_with_logdevio
     with_timezone('utc') do
-      rotate_age = 2
-      rotate_size = 100
-      path = "#{TMP_DIR}/log-dev-io-#{rotate_size}-#{rotate_age}"
-      path0 = path + '.0'
-      path1 = path + '.1'
+      begin
+        rotate_age = 2
+        rotate_size = 100
+        path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
+        path0 = path + '.0'
+        path1 = path + '.1'
+
+        logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
+        logger = ServerEngine::DaemonLogger.new(logdev)
+        log = Fluent::Log.new(logger)
+
+        msg = 'a' * 101
+        log.info msg
+        assert_match msg, File.read(path)
+        assert_true File.exist?(path)
+        assert_true !File.exist?(path0)
+        assert_true !File.exist?(path1)
+
+        # create log.0
+        msg2 = 'b' * 101
+        log.info msg2
+        c = File.read(path)
+        c0 = File.read(path0)
+        assert_match msg2, c
+        assert_match msg, c0
+        assert_true File.exist?(path)
+        assert_true File.exist?(path0)
+        assert_true !File.exist?(path1)
+
+        # rotate
+        msg3 = 'c' * 101
+        log.info msg3
+        c = File.read(path)
+        c0 = File.read(path0)
+        assert_match msg3, c
+        assert_match msg2, c0
+        assert_true File.exist?(path)
+        assert_true File.exist?(path0)
+        assert_true !File.exist?(path1)
+      ensure
+       logdev&.close
+     end
+    end
+  end
 
-      logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
-      logger = ServerEngine::DaemonLogger.new(logdev)
-      log = Fluent::Log.new(logger)
+  def test_reopen
+    path = Pathname(@tmp_dir) + "fluent.log"
 
-      msg = 'a' * 101
-      log.info msg
-      assert_match msg, File.read(path)
-      assert_true File.exist?(path)
-      assert_true !File.exist?(path0)
-      assert_true !File.exist?(path1)
+    logdev = Fluent::LogDeviceIO.new(path.to_s)
+    logger = ServerEngine::DaemonLogger.new(logdev)
+    log = Fluent::Log.new(logger, path: path)
 
-      # create log.0
-      msg2 = 'b' * 101
-      log.info msg2
-      c = File.read(path)
-      c0 = File.read(path0)
-      assert_match msg2, c
-      assert_match msg, c0
-      assert_true File.exist?(path)
-      assert_true File.exist?(path0)
-      assert_true !File.exist?(path1)
-
-      # rotate
-      msg3 = 'c' * 101
-      log.info msg3
-      c = File.read(path)
-      c0 = File.read(path0)
-      assert_match msg3, c
-      assert_match msg2, c0
-      assert_true File.exist?(path)
-      assert_true File.exist?(path0)
-      assert_true !File.exist?(path1)
-    end
+    message = "This is test message."
+
+    log.info message
+    log.reopen!
+    log.info message
+
+    assert { path.read.lines.count{ |line| line.include?(message) } == 2 }
+    # Assert reopening the same file.
+    # Especially, on Windows, the filepath is fixed for each process with rotate,
+    # so we need to care about this.
+    assert { path.parent.entries.size == 3 } # [".", "..", "fluent.log"]
+  ensure
+    logdev&.close
   end
 end
 

data/test/test_msgpack_factory.rb

@@ -15,4 +15,36 @@ class MessagePackFactoryTest < Test::Unit::TestCase
     assert mp.msgpack_factory
     assert mp.msgpack_factory
   end
+
+  sub_test_case 'thread_local_msgpack_packer' do
+    test 'packer is cached' do
+      packer1 = Fluent::MessagePackFactory.thread_local_msgpack_packer
+      packer2 = Fluent::MessagePackFactory.thread_local_msgpack_packer
+      assert_equal packer1, packer2
+    end
+  end
+
+  sub_test_case 'thread_local_msgpack_unpacker' do
+    test 'unpacker is cached' do
+      unpacker1 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
+      unpacker2 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
+      assert_equal unpacker1, unpacker2
+    end
+
+    # We need to reset the buffer every time so that received incomplete data
+    # must not affect data from other senders.
+    test 'reset the internal buffer of unpacker every time' do
+      unpacker1 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
+      unpacker1.feed_each("\xA6foo") do |result|
+        flunk("This callback must not be called since the data is uncomplete.")
+      end
+
+      records = []
+      unpacker2 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
+      unpacker2.feed_each("\xA3foo") do |result|
+        records.append(result)
+      end
+      assert_equal ["foo"], records
+    end
+  end
 end

data/test/test_plugin_classes.rb

@@ -5,11 +5,78 @@ require 'fluent/plugin/bare_output'
 require 'fluent/plugin/filter'
 
 module FluentTest
+  class FluentTestCounterMetrics < Fluent::Plugin::Metrics
+    Fluent::Plugin.register_metrics('test_counter', self)
+
+    attr_reader :data
+
+    def initialize
+      super
+      @data = 0
+    end
+    def get
+      @data
+    end
+    def inc
+      @data +=1
+    end
+    def add(value)
+      @data += value
+    end
+    def set(value)
+      @data = value
+    end
+    def close
+      @data = 0
+      super
+    end
+  end
+
+  class FluentTestGaugeMetrics < Fluent::Plugin::Metrics
+    Fluent::Plugin.register_metrics('test_gauge', self)
+
+    attr_reader :data
+
+    def initialize
+      super
+      @data = 0
+    end
+    def get
+      @data
+    end
+    def inc
+      @data += 1
+    end
+    def dec
+      @data -=1
+    end
+    def add(value)
+      @data += value
+    end
+    def sub(value)
+      @data -= value
+    end
+    def set(value)
+      @data = value
+    end
+    def close
+      @data = 0
+      super
+    end
+  end
+
   class FluentTestInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input('test_in', self)
 
     attr_reader :started
 
+    def initialize
+      super
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def start
       super
       @started = true
@@ -28,6 +95,13 @@ module FluentTest
 
     config_param :num, :integer, default: 10000
 
+    def initialize
+      super
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def start
       super
       @started = true
@@ -49,6 +123,15 @@ module FluentTest
     def initialize
       super
       @events = Hash.new { |h, k| h[k] = [] }
+      # stub metrics instances
+      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :events
@@ -168,6 +251,19 @@ module FluentTest
   class FluentTestErrorOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_error', self)
 
+    def initialize
+      super
+      # stub metrics instances
+      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def format(tag, time, record)
       raise "emit error!"
     end
@@ -184,6 +280,9 @@ module FluentTest
       super()
       @num = 0
       @field = field
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :num
@@ -213,6 +312,9 @@ module FluentTest
       super()
       @num = 0
       @field = field
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :num

data/test/test_root_agent.rb

@@ -16,7 +16,8 @@ class RootAgentTest < ::Test::Unit::TestCase
 
   data(
     'suppress interval' => [{'emit_error_log_interval' => 30}, {:@suppress_emit_error_log_interval => 30}],
-    'without source' => [{'without_source' => true}, {:@without_source => true}]
+    'without source' => [{'without_source' => true}, {:@without_source => true}],
+    'enable input metrics' => [{'enable_input_metrics' => true}, {:@enable_input_metrics => true}],
     )
   def test_initialize_with_opt(data)
     opt, expected = data
@@ -109,6 +110,34 @@ EOC
       end
     end
 
+    test 'raises configuration error for label without name' do
+      conf = <<-EOC
+<label>
+  @type test_out
+</label>
+EOC
+      errmsg = "Missing symbol argument on <label> directive"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for <label @ROOT>' do
+      conf = <<-EOC
+<source>
+  @type test_in
+  @label @ROOT
+</source>
+<label @ROOT>
+  @type test_out
+</label>
+EOC
+      errmsg = "@ROOT for <label> is not permitted, reserved for getting root router"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
     test 'raises configuration error if there are not match sections in label section' do
       conf = <<-EOC
 <source>