fluentd 1.15.2-x86-mingw32 → 1.16.1-x86-mingw32

data/test/plugin/test_in_tcp.rb

@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal hostname, event[2]['host']
   end
 
+  test "send_keepalive_packet_can_be_enabled" do
+    d = create_driver(base_config + %!
+      format none
+      send_keepalive_packet true
+    !)
+    assert_true d.instance.send_keepalive_packet
+
+    d = create_driver(base_config + %!
+      format none
+    !)
+    assert_false d.instance.send_keepalive_packet
+  end
+
   test 'source_address_key' do
     d = create_driver(base_config + %!
       format none
@@ -205,13 +218,13 @@ class TcpInputTest < Test::Unit::TestCase
           </client>
         </security>
       !)
-      d.run(shutdown: false, expect_records: 1, timeout: 2) do
+      d.run(expect_records: 1, timeout: 2) do
         create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send("hello\n", 0)
         end
       end
 
-      assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
+      assert_equal 1, d.logs.count { |l| l =~ /anonymous client/ }
       assert_equal 0, d.events.size
     end
   end
@@ -240,4 +253,76 @@ class TcpInputTest < Test::Unit::TestCase
       assert_equal 'hello', event[2]['msg']
     end
   end
+
+  sub_test_case "message_length_limit" do
+    data("batch_emit", { extract: "" }, keep: true)
+    data("single_emit", { extract: "<extract>\ntag_key tag\n</extract>\n" }, keep: true)
+    test "drop records exceeding limit" do |data|
+      message_length_limit = 10
+      d = create_driver(base_config + %!
+        message_length_limit #{message_length_limit}
+        <parse>
+          @type none
+        </parse>
+        #{data[:extract]}
+      !)
+      d.run(expect_records: 2, timeout: 10) do
+        create_tcp_socket('127.0.0.1', @port) do |sock|
+          sock.send("a" * message_length_limit + "\n", 0)
+          sock.send("b" * (message_length_limit + 1) + "\n", 0)
+          sock.send("c" * (message_length_limit - 1) + "\n", 0)
+        end
+      end
+
+      expected_records = [
+        "a" * message_length_limit,
+        "c" * (message_length_limit - 1)
+      ]
+      actual_records = d.events.collect do |event|
+        event[2]["message"]
+      end
+
+      assert_equal expected_records, actual_records
+    end
+
+    test "clear buffer and discard the subsequent data until the next delimiter" do |data|
+      message_length_limit = 12
+      d = create_driver(base_config + %!
+        message_length_limit #{message_length_limit}
+        delimiter ";"
+        <parse>
+          @type json
+        </parse>
+        #{data[:extract]}
+      !)
+      d.run(expect_records: 1, timeout: 10) do
+        create_tcp_socket('127.0.0.1', @port) do |sock|
+          sock.send('{"message":', 0)
+          sock.send('"hello', 0)
+          sleep 1 # To make the server read data and clear the buffer here.
+          sock.send('world!"};', 0) # This subsequent data must be discarded so that a parsing failure doesn't occur.
+          sock.send('{"k":"v"};', 0) # This will succeed to parse.
+        end
+      end
+
+      logs = d.logs.collect do |log|
+        log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
+      end
+      actual_records = d.events.collect do |event|
+        event[2]
+      end
+
+      assert_equal(
+        {
+          # Asserting that '[warn]: pattern not matched message="world!\"}"' warning does not occur.
+          logs: ['[info]: The buffer size exceeds \'message_length_limit\', cleared: limit=12 size=17 head="{\"message\":\"hello"' + "\n"],
+          records: [{"k" => "v"}],
+        },
+        {
+          logs: logs[1..],
+          records: actual_records,
+        }
+      )
+    end
+  end
 end

data/test/plugin/test_in_udp.rb

@@ -265,4 +265,32 @@ class UdpInputTest < Test::Unit::TestCase
       end
     end
   end
+
+  test 'message_length_limit' do
+    message_length_limit = 32
+    d = create_driver(base_config + %!
+      format none
+      message_length_limit #{message_length_limit}
+    !)
+    d.run(expect_records: 3) do
+      create_udp_socket('127.0.0.1', @port) do |u|
+        3.times do |i|
+          u.send("#{i}" * 40 + "\n", 0)
+        end
+      end
+    end
+
+    if Fluent.windows?
+      expected_records = []
+    else
+      expected_records = 3.times.collect do |i|
+        "#{i}" * message_length_limit
+      end
+    end
+    actual_records = d.events.collect do |event|
+      event[2]["message"]
+    end
+
+    assert_equal expected_records, actual_records
+  end
 end

data/test/plugin/test_out_file.rb

@@ -5,6 +5,7 @@ require 'fileutils'
 require 'time'
 require 'timecop'
 require 'zlib'
+require 'fluent/file_wrapper'
 
 class FileOutputTest < Test::Unit::TestCase
   def setup
@@ -1016,7 +1017,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'returns filepath with index which does not exist yet' do
       5.times do |i|
-        File.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
+        Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
       end
       @i.find_filepath_available(File.join(@tmp, "exist_**.log")) do |path|
         assert_equal File.join(@tmp, "exist_5.log"), path
@@ -1025,7 +1026,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'creates lock directory when with_lock is true to exclude operations of other worker process' do
       5.times do |i|
-        File.open(File.join(@tmp, "exist_#{i}.log"), 'a')
+        Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a')
       end
       Dir.mkdir(File.join(@tmp, "exist_5.log.lock"))
       @i.find_filepath_available(File.join(@tmp, "exist_**.log"), with_lock: true) do |path|

data/test/plugin/test_out_forward.rb

@@ -1331,26 +1331,22 @@ EOL
         d = create_driver(output_conf)
         d.instance_start
 
-        begin
-          chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-          mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                                   linger_timeout: anything,
-                                                   send_timeout: anything,
-                                                   recv_timeout: anything,
-                                                   connect_timeout: anything) { |sock|
-            mock(sock).close.once; sock
-          }.twice
-
-          target_input_driver.run(timeout: 15) do
-            d.run(shutdown: false) do
-              node = d.instance.nodes.first
-              2.times do
-                node.send_data('test', chunk) rescue nil
-              end
+        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+        mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                                 linger_timeout: anything,
+                                                 send_timeout: anything,
+                                                 recv_timeout: anything,
+                                                 connect_timeout: anything) { |sock|
+          mock(sock).close.once; sock
+        }.twice
+
+        target_input_driver.run(timeout: 15) do
+          d.run do
+            node = d.instance.nodes.first
+            2.times do
+              node.send_data('test', chunk) rescue nil
             end
           end
-        ensure
-          d.instance_shutdown
         end
       end
     end

data/test/plugin/test_out_http.rb

@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
           password hello?
         </auth>
       ])
+      d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
       d.run(default_tag: 'test.http', shutdown: false) do
         test_events.each { |event|
           d.feed(event)

data/test/plugin/test_output.rb

@@ -803,7 +803,10 @@ class OutputTest < Test::Unit::TestCase
     end
 
     test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
+      tmp_dir = File.join(__dir__, '../tmp/test_output')
+
       i = create_output(:delayed)
+      i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
       try_write_called = false
       i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
 
@@ -820,6 +823,8 @@ class OutputTest < Test::Unit::TestCase
       assert try_write_called
 
       i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    ensure
+      FileUtils.rm_rf(tmp_dir)
     end
 
     test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
@@ -849,7 +854,10 @@ class OutputTest < Test::Unit::TestCase
     end
 
     test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
+      tmp_dir = File.join(__dir__, '../tmp/test_output')
+
       i = create_output(:full)
+      i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
       write_called = false
       try_write_called = false
       i.register(:write){ |chunk| write_called = true }
@@ -872,6 +880,8 @@ class OutputTest < Test::Unit::TestCase
       assert try_write_called
 
       i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    ensure
+      FileUtils.rm_rf(tmp_dir)
     end
 
     test 'flush_interval is ignored when flush_mode is not interval' do
@@ -1062,4 +1072,263 @@ class OutputTest < Test::Unit::TestCase
       }
     end
   end
+
+  sub_test_case "actual_flush_thread_count" do
+    data(
+      "Not buffered",
+      {
+        output_type: :sync,
+        config: config_element(),
+        expected: 0,
+      }
+    )
+    data(
+      "Buffered with singile thread",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        expected: 1,
+      }
+    )
+    data(
+      "Buffered with multiple threads",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        expected: 8,
+      }
+    )
+    test "actual_flush_thread_count" do |data|
+      o = create_output(data[:output_type])
+      o.configure(data[:config])
+      assert_equal data[:expected], o.actual_flush_thread_count
+    end
+
+    data(
+      "Buffered with single thread",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}), 
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        expected: 1,
+      }
+    )
+    data(
+      "Buffered with multiple threads",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        expected: 8,
+      }
+    )
+    test "actual_flush_thread_count for secondary" do |data|
+      primary = create_output(data[:output_type])
+      primary.configure(data[:config])
+      assert_equal data[:expected], primary.secondary.actual_flush_thread_count
+    end
+  end
+
+  sub_test_case "synchronize_path" do
+    def setup
+      Dir.mktmpdir do |lock_dir|
+        ENV['FLUENTD_LOCK_DIR'] = lock_dir
+        yield
+      end
+    end
+
+    def assert_worker_lock(lock_path, expect_locked)
+      # With LOCK_NB set, flock() returns:
+      #   * `false` when the file is already locked.
+      #   * `0` when the file is not locked.
+      File.open(lock_path, "w") do |f|
+        if expect_locked
+          assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
+        else
+          assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
+        end
+      end
+    end
+
+    def assert_thread_lock(output_plugin, expect_locked)
+      t = Thread.new do
+        output_plugin.synchronize_path("test") do
+        end
+      end
+      if expect_locked
+        assert_nil t.join(3)
+      else
+        assert_not_nil t.join(3)
+      end
+    end
+
+    data(
+      "Not buffered with single worker",
+      {
+        output_type: :sync,
+        config: config_element(),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Not buffered with multiple workers",
+      {
+        output_type: :sync,
+        config: config_element(),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with single thread and single worker",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and single worker",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: true,
+      }
+    )
+    data(
+      "Buffered with single thread and multiple workers",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and multiple workers",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: true,
+      }
+    )
+    test "synchronize_path" do |data|
+      o = create_output(data[:output_type])
+      o.configure(data[:config])
+      o.system_config_override(workers: data[:workers])
+
+      test_lock_name = "test_lock_name"
+      lock_path = o.get_lock_path(test_lock_name)
+
+      o.synchronize_path(test_lock_name) do
+        assert_worker_lock(lock_path, data[:expect_worker_lock])
+        assert_thread_lock(o, data[:expect_thread_lock])
+      end
+
+      assert_worker_lock(lock_path, false)
+      assert_thread_lock(o, false)
+    end
+
+    data(
+      "Buffered with single thread and single worker",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and single worker",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: true,
+      }
+    )
+    data(
+      "Buffered with single thread and multiple workers",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and multiple workers",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: true,
+      }
+    )
+    test "synchronize_path for secondary" do |data|
+      primary = create_output(data[:output_type])
+      primary.configure(data[:config])
+      secondary = primary.secondary
+      secondary.system_config_override(workers: data[:workers])
+
+      test_lock_name = "test_lock_name"
+      lock_path = secondary.get_lock_path(test_lock_name)
+
+      secondary.synchronize_path(test_lock_name) do
+        assert_worker_lock(lock_path, data[:expect_worker_lock])
+        assert_thread_lock(secondary, data[:expect_thread_lock])
+      end
+
+      assert_worker_lock(lock_path, false)
+      assert_thread_lock(secondary, false)
+    end
+  end
 end

data/test/plugin/test_output_as_buffered_compress.rb

@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
       @format ? @format.call(tag, time, record) : [tag, time, record].to_json
     end
   end
+
+  def self.dummy_event_stream
+    Fluent::ArrayEventStream.new(
+      [
+        [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
+        [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
+        [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
+      ]
+    )
+  end
 end
 
 class BufferedOutputCompressTest < Test::Unit::TestCase
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
     end
   end
 
-  def dummy_event_stream
-    Fluent::ArrayEventStream.new(
-      [
-        [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
-        [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
-        [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
-      ]
-    )
-  end
-
   TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
 
   setup do
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
   end
 
   data(
-    handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_stream_with_standard_format:  config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_stream_with_standard_format_and_file_chunk:  config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+    :buffer_config,
+    [
+      config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+      config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+      config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+      config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+    ],
   )
-  test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
+  data(
+    :input_es,
+    [
+      FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
+      # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
+      # https://github.com/fluent/fluentd/issues/4146
+      Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
+    ],
+  )
+  test 'call a standard format when output plugin adds data to chunk' do |data|
+    buffer_config = data[:buffer_config]
+    es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
+
     @i = create_output(:async)
     @i.configure(config_element('ROOT','', {}, [buffer_config]))
     @i.start
     @i.after_start
 
     io = StringIO.new
-    es = dummy_event_stream
-    expected = es.map { |e| e }
+    expected = es.dup.map { |t, r| [t, r] }
     compressed_data = ''
 
     assert_equal :gzip, @i.buffer.compress
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
     @i.after_start
 
     io = StringIO.new
-    es = dummy_event_stream
+    es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
     expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
     compressed_data = ''
 

data/test/plugin/test_parser_regexp.rb

@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
       if initialize_conf
         Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
       else
-        # Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
-        instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
-        instance.configure(conf)
-        d = Struct.new(:instance).new
-        d.instance = instance
-        d
+        Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
       end
     end
 

data/test/plugin_helper/test_http_server_helper.rb

@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
     end
 
     client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
-    reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
+    reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
 
     resp = nil
     error = nil