fluentd222 1.16.2-x86_64-linux

data/test/log/test_console_adapter.rb

@@ -0,0 +1,110 @@
+require_relative '../helper'
+
+require 'fluent/log'
+require 'fluent/log/console_adapter'
+
+class ConsoleAdapterTest < Test::Unit::TestCase
+  def setup
+    @timestamp = Time.parse("2023-01-01 15:32:41 +0000")
+    @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
+    Timecop.freeze(@timestamp)
+
+    @logdev = Fluent::Test::DummyLogDevice.new
+    @logger = ServerEngine::DaemonLogger.new(@logdev)
+    @fluent_log = Fluent::Log.new(@logger)
+    @console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
+  end
+
+  def teardown
+    Timecop.return
+  end
+
+  def test_expected_log_levels
+    assert_equal({debug: 0, info: 1, warn: 2, error: 3, fatal: 4},
+                 Console::Logger::LEVELS)
+  end
+
+  data(trace: [Fluent::Log::LEVEL_TRACE, :debug],
+       debug: [Fluent::Log::LEVEL_DEBUG, :debug],
+       info: [Fluent::Log::LEVEL_INFO, :info],
+       warn: [Fluent::Log::LEVEL_WARN, :warn],
+       error: [Fluent::Log::LEVEL_ERROR, :error],
+       fatal: [Fluent::Log::LEVEL_FATAL, :fatal])
+  def test_reflect_log_level(data)
+    level, expected = data
+    @fluent_log.level = level
+    console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
+    assert_equal(Console::Logger::LEVELS[expected],
+                 console_logger.level)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_string_subject(level)
+    @console_logger.send(level, "subject")
+    assert_equal(["#{@timestamp_str} [#{level}]:   0.0s: subject\n"],
+                 @logdev.logs)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_args(level)
+    @console_logger.send(level, "subject", 1, 2, 3)
+    assert_equal([
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
+                   "      | 1\n" +
+                   "      | 2\n" +
+                   "      | 3\n"
+                 ],
+                 @logdev.logs)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_options(level)
+    @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
+    assert_equal([
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
+                   "      | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
+                 ],
+                 @logdev.logs)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_block(level)
+    @console_logger.send(level, "subject") { "block message" }
+    assert_equal([
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
+                   "      | block message\n"
+                 ],
+                 @logdev.logs)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_multiple_entries(level)
+    @console_logger.send(level, "subject1")
+    @console_logger.send(level, "line2")
+    assert_equal([
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject1\n",
+                   "#{@timestamp_str} [#{level}]:   0.0s: line2\n"
+                 ],
+                 @logdev.logs)
+  end
+end

data/test/plugin/data/sd_file/config

@@ -0,0 +1,11 @@
+- 'host': 127.0.0.1
+  'port': 24224
+  'weight': 1
+  'name': test1
+  'standby': false
+  'username': user1
+  'password': pass1
+  'shared_key': key1
+- 'host': 127.0.0.1
+  'port': 24225
+  'weight': 1

data/test/plugin/data/sd_file/config.json

@@ -0,0 +1,17 @@
+[
+    {
+        "host": "127.0.0.1",
+        "port": 24224,
+        "weight": 1,
+        "name": "test1",
+        "standby": false,
+        "username": "user1",
+        "password": "pass1",
+        "shared_key": "key1"
+    },
+    {
+        "host": "127.0.0.1",
+        "port": 24225,
+        "weight": 1
+    }
+]

data/test/plugin/data/sd_file/config.yaml

@@ -0,0 +1,11 @@
+- 'host': 127.0.0.1
+  'port': 24224
+  'weight': 1
+  'name': test1
+  'standby': false
+  'username': user1
+  'password': pass1
+  'shared_key': key1
+- 'host': 127.0.0.1
+  'port': 24225
+  'weight': 1

data/test/plugin/data/sd_file/config.yml

@@ -0,0 +1,11 @@
+- 'host': 127.0.0.1
+  'port': 24224
+  'weight': 1
+  'name': test1
+  'standby': false
+  'username': user1
+  'password': pass1
+  'shared_key': key1
+- 'host': 127.0.0.1
+  'port': 24225
+  'weight': 1

data/test/plugin/data/sd_file/invalid_config.yml

@@ -0,0 +1,7 @@
+- 'host': 127.0.0.1
+  'weight': 1
+  'name': test1
+  'standby': false
+  'username': user1
+  'password': pass1
+  'shared_key': key1

data/test/plugin/in_tail/test_fifo.rb

@@ -0,0 +1,121 @@
+require_relative '../../helper'
+
+require 'fluent/plugin/in_tail'
+
+class IntailFIFO < Test::Unit::TestCase
+  sub_test_case '#read_line' do
+    test 'returns lines spliting per `\n`' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    test 'concant line when line is separated' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3 + 'test').force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      fifo << "2\n"
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n", "test2\n"], lines
+    end
+
+    test 'returns lines which convert encoding' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::UTF_8)
+      text = ("test\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::UTF_8, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    test 'reads lines as from_encoding' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::UTF_8)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    sub_test_case 'when it includes multi byte chars' do
+      test 'handles it as ascii_8bit' do
+        fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+        text = ("てすと\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+        fifo << text
+        lines = []
+        fifo.read_lines(lines)
+        assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+        assert_equal ["てすと\n", "てすと\n", "てすと\n"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines
+      end
+
+      test 'replaces character with ? when convert error happens' do
+        fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, Encoding::ASCII_8BIT)
+        text = ("てすと\n" * 3).force_encoding(Encoding::UTF_8)
+        fifo << text
+        lines = []
+        fifo.read_lines(lines)
+        assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+        assert_equal ["???\n", "???\n", "???\n"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines
+      end
+    end
+
+    test 'reutrns nothing when buffer is empty' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal [], lines
+
+      text = "test\n" * 3
+      fifo << text
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal [], lines
+    end
+  end
+
+  sub_test_case '#<<' do
+    test 'does not make any change about encoding to an argument' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::UTF_8)
+
+      assert_equal Encoding::UTF_8, text.encoding
+      fifo << text
+      assert_equal Encoding::UTF_8, text.encoding
+    end
+  end
+
+  sub_test_case '#bytesize' do
+    test 'reutrns buffer size' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = "test\n" * 3 + 'test'
+      fifo << text
+
+      assert_equal text.bytesize, fifo.bytesize
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      assert_equal 'test'.bytesize, fifo.bytesize
+      fifo << "2\n"
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n", "test2\n"], lines
+
+      assert_equal 0, fifo.bytesize
+    end
+  end
+end

data/test/plugin/in_tail/test_io_handler.rb

@@ -0,0 +1,150 @@
+require_relative '../../helper'
+
+require 'fluent/plugin/in_tail'
+require 'fluent/plugin/metrics_local'
+require 'tempfile'
+
+class IntailIOHandlerTest < Test::Unit::TestCase
+  setup do
+    @file = Tempfile.new('intail_io_handler').binmode
+    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+    opened_file_metrics.configure(config_element('metrics', '', {}))
+    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+    closed_file_metrics.configure(config_element('metrics', '', {}))
+    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+    rotated_file_metrics.configure(config_element('metrics', '', {}))
+    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+  end
+
+  teardown do
+    @file.close rescue nil
+    @file.unlink rescue nil
+  end
+
+  def create_target_info
+    Fluent::Plugin::TailInput::TargetInfo.new(@file.path, Fluent::FileWrapper.stat(@file.path).ino)
+  end
+
+  def create_watcher
+    Fluent::Plugin::TailInput::TailWatcher.new(create_target_info, nil, nil, nil, nil, nil, nil, nil, nil)
+  end
+
+  test '#on_notify load file content and passed it to receive_lines method' do
+    text = "this line is test\ntest line is test\n"
+    @file.write(text)
+    @file.close
+
+    watcher = create_watcher
+
+    update_pos = 0
+
+    stub(watcher).pe do
+      pe = 'position_file'
+      stub(pe).read_pos { 0 }
+      stub(pe).update_pos { |val| update_pos = val }
+      pe
+    end
+
+    returned_lines = ''
+    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
+      returned_lines << lines.join
+      true
+    end
+
+    r.on_notify
+    assert_equal text.bytesize, update_pos
+    assert_equal text, returned_lines
+
+    r.on_notify
+
+    assert_equal text.bytesize, update_pos
+    assert_equal text, returned_lines
+  end
+
+  sub_test_case 'when open_on_every_update is true and read_pos returns always 0' do
+    test 'open new IO and change pos to 0 and read it' do
+      text = "this line is test\ntest line is test\n"
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = create_watcher
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = ''
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true, metrics: @metrics) do |lines, _watcher|
+        returned_lines << lines.join
+        true
+      end
+
+      r.on_notify
+      assert_equal text.bytesize, update_pos
+      assert_equal text, returned_lines
+
+      r.on_notify
+      assert_equal text * 2, returned_lines
+    end
+  end
+
+  sub_test_case 'when limit is 5' do
+    test 'call receive_lines once when short line(less than 8192)' do
+      text = "line\n" * 8
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = create_watcher
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = []
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
+        returned_lines << lines.dup
+        true
+      end
+
+      r.on_notify
+      assert_equal text.bytesize, update_pos
+      assert_equal 8, returned_lines[0].size
+    end
+
+    test 'call receive_lines some times when long line(more than 8192)' do
+      t = 'line' * (8192 / 8)
+      text = "#{t}\n" * 8
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = create_watcher
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = []
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
+        returned_lines << lines.dup
+        true
+      end
+
+      r.on_notify
+      assert_equal text.bytesize, update_pos
+      assert_equal 5, returned_lines[0].size
+      assert_equal 3, returned_lines[1].size
+    end
+  end
+end