fluentd 1.15.3-x86-mingw32 → 1.16.1-x86-mingw32

data/test/plugin/test_output.rb

@@ -803,7 +803,10 @@ class OutputTest < Test::Unit::TestCase
     end
 
     test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
+      tmp_dir = File.join(__dir__, '../tmp/test_output')
+
       i = create_output(:delayed)
+      i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
       try_write_called = false
       i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
 
@@ -820,6 +823,8 @@ class OutputTest < Test::Unit::TestCase
       assert try_write_called
 
       i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    ensure
+      FileUtils.rm_rf(tmp_dir)
     end
 
     test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
@@ -849,7 +854,10 @@ class OutputTest < Test::Unit::TestCase
     end
 
     test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
+      tmp_dir = File.join(__dir__, '../tmp/test_output')
+
       i = create_output(:full)
+      i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
       write_called = false
       try_write_called = false
       i.register(:write){ |chunk| write_called = true }
@@ -872,6 +880,8 @@ class OutputTest < Test::Unit::TestCase
       assert try_write_called
 
       i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    ensure
+      FileUtils.rm_rf(tmp_dir)
     end
 
     test 'flush_interval is ignored when flush_mode is not interval' do
@@ -1062,4 +1072,263 @@ class OutputTest < Test::Unit::TestCase
       }
     end
   end
+
+  sub_test_case "actual_flush_thread_count" do
+    data(
+      "Not buffered",
+      {
+        output_type: :sync,
+        config: config_element(),
+        expected: 0,
+      }
+    )
+    data(
+      "Buffered with singile thread",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        expected: 1,
+      }
+    )
+    data(
+      "Buffered with multiple threads",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        expected: 8,
+      }
+    )
+    test "actual_flush_thread_count" do |data|
+      o = create_output(data[:output_type])
+      o.configure(data[:config])
+      assert_equal data[:expected], o.actual_flush_thread_count
+    end
+
+    data(
+      "Buffered with single thread",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}), 
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        expected: 1,
+      }
+    )
+    data(
+      "Buffered with multiple threads",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        expected: 8,
+      }
+    )
+    test "actual_flush_thread_count for secondary" do |data|
+      primary = create_output(data[:output_type])
+      primary.configure(data[:config])
+      assert_equal data[:expected], primary.secondary.actual_flush_thread_count
+    end
+  end
+
+  sub_test_case "synchronize_path" do
+    def setup
+      Dir.mktmpdir do |lock_dir|
+        ENV['FLUENTD_LOCK_DIR'] = lock_dir
+        yield
+      end
+    end
+
+    def assert_worker_lock(lock_path, expect_locked)
+      # With LOCK_NB set, flock() returns:
+      #   * `false` when the file is already locked.
+      #   * `0` when the file is not locked.
+      File.open(lock_path, "w") do |f|
+        if expect_locked
+          assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
+        else
+          assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
+        end
+      end
+    end
+
+    def assert_thread_lock(output_plugin, expect_locked)
+      t = Thread.new do
+        output_plugin.synchronize_path("test") do
+        end
+      end
+      if expect_locked
+        assert_nil t.join(3)
+      else
+        assert_not_nil t.join(3)
+      end
+    end
+
+    data(
+      "Not buffered with single worker",
+      {
+        output_type: :sync,
+        config: config_element(),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Not buffered with multiple workers",
+      {
+        output_type: :sync,
+        config: config_element(),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with single thread and single worker",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and single worker",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: true,
+      }
+    )
+    data(
+      "Buffered with single thread and multiple workers",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and multiple workers",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: true,
+      }
+    )
+    test "synchronize_path" do |data|
+      o = create_output(data[:output_type])
+      o.configure(data[:config])
+      o.system_config_override(workers: data[:workers])
+
+      test_lock_name = "test_lock_name"
+      lock_path = o.get_lock_path(test_lock_name)
+
+      o.synchronize_path(test_lock_name) do
+        assert_worker_lock(lock_path, data[:expect_worker_lock])
+        assert_thread_lock(o, data[:expect_thread_lock])
+      end
+
+      assert_worker_lock(lock_path, false)
+      assert_thread_lock(o, false)
+    end
+
+    data(
+      "Buffered with single thread and single worker",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and single worker",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: true,
+      }
+    )
+    data(
+      "Buffered with single thread and multiple workers",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and multiple workers",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: true,
+      }
+    )
+    test "synchronize_path for secondary" do |data|
+      primary = create_output(data[:output_type])
+      primary.configure(data[:config])
+      secondary = primary.secondary
+      secondary.system_config_override(workers: data[:workers])
+
+      test_lock_name = "test_lock_name"
+      lock_path = secondary.get_lock_path(test_lock_name)
+
+      secondary.synchronize_path(test_lock_name) do
+        assert_worker_lock(lock_path, data[:expect_worker_lock])
+        assert_thread_lock(secondary, data[:expect_thread_lock])
+      end
+
+      assert_worker_lock(lock_path, false)
+      assert_thread_lock(secondary, false)
+    end
+  end
 end

data/test/plugin/test_output_as_buffered_compress.rb

@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
       @format ? @format.call(tag, time, record) : [tag, time, record].to_json
     end
   end
+
+  def self.dummy_event_stream
+    Fluent::ArrayEventStream.new(
+      [
+        [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
+        [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
+        [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
+      ]
+    )
+  end
 end
 
 class BufferedOutputCompressTest < Test::Unit::TestCase
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
     end
   end
 
-  def dummy_event_stream
-    Fluent::ArrayEventStream.new(
-      [
-        [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
-        [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
-        [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
-      ]
-    )
-  end
-
   TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
 
   setup do
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
   end
 
   data(
-    handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_stream_with_standard_format:  config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_stream_with_standard_format_and_file_chunk:  config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+    :buffer_config,
+    [
+      config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+      config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+      config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+      config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+    ],
   )
-  test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
+  data(
+    :input_es,
+    [
+      FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
+      # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
+      # https://github.com/fluent/fluentd/issues/4146
+      Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
+    ],
+  )
+  test 'call a standard format when output plugin adds data to chunk' do |data|
+    buffer_config = data[:buffer_config]
+    es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
+
     @i = create_output(:async)
     @i.configure(config_element('ROOT','', {}, [buffer_config]))
     @i.start
     @i.after_start
 
     io = StringIO.new
-    es = dummy_event_stream
-    expected = es.map { |e| e }
+    expected = es.dup.map { |t, r| [t, r] }
     compressed_data = ''
 
     assert_equal :gzip, @i.buffer.compress
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
     @i.after_start
 
     io = StringIO.new
-    es = dummy_event_stream
+    es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
     expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
     compressed_data = ''
 

data/test/plugin/test_parser_regexp.rb

@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
       if initialize_conf
         Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
       else
-        # Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
-        instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
-        instance.configure(conf)
-        d = Struct.new(:instance).new
-        d.instance = instance
-        d
+        Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
       end
     end
 

data/test/plugin_helper/test_http_server_helper.rb

@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
     end
 
     client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
-    reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
+    reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
 
     resp = nil
     error = nil

data/test/plugin_helper/test_server.rb

@@ -16,14 +16,20 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
   setup do
     @port = unused_port
-    @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
-      FileUtils.rm_f @socket_manager_path
+    if Fluent.windows?
+      @socket_manager_server = ServerEngine::SocketManager::Server.open
+      @socket_manager_path = @socket_manager_server.path
+    else
+      @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+        FileUtils.rm_f @socket_manager_path
+      end
+      @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
     end
-    @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
     ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
 
     @d = Dummy.new
+    @d.under_plugin_development = true
     @d.start
     @d.after_start
   end
@@ -37,7 +43,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
     (@d.terminated? || @d.terminate) rescue nil
 
     @socket_manager_server.close
-    if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
       FileUtils.rm_f @socket_manager_path
     end
   end
@@ -789,6 +795,50 @@ class ServerPluginHelperTest < Test::Unit::TestCase
         end
       end
     end
+
+    sub_test_case 'over max_bytes' do
+      data("cut off on Non-Windows", { max_bytes: 32, records: ["a" * 40], expected: ["a" * 32] }, keep: true) unless Fluent.windows?
+      data("drop on Windows", { max_bytes: 32, records: ["a" * 40], expected: [] }, keep: true) if Fluent.windows?
+      test 'with sock' do |data|
+        max_bytes, records, expected = data.values
+
+        actual_records = []
+        @d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data, sock|
+          actual_records << data
+        end
+
+        open_client(:udp, "127.0.0.1", @port) do |sock|
+          records.each do |record|
+            sock.send(record, 0)
+          end
+        end
+
+        waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
+        sleep 1 if expected.size == 0 # To confirm no record recieved.
+
+        assert_equal expected, actual_records
+      end
+
+      test 'without sock' do |data|
+        max_bytes, records, expected = data.values
+
+        actual_records = []
+        @d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data|
+          actual_records << data
+        end
+
+        open_client(:udp, "127.0.0.1", @port) do |sock|
+          records.each do |record|
+            sock.send(record, 0)
+          end
+        end
+
+        waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
+        sleep 1 if expected.size == 0 # To confirm no record recieved.
+
+        assert_equal expected, actual_records
+      end
+    end
   end
 
   module CertUtil
@@ -1570,6 +1620,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
   def open_client(proto, addr, port)
     client = case proto
+             when :udp
+               c = UDPSocket.open
+               c.connect(addr, port)
+               c
              when :tcp
                TCPSocket.open(addr, port)
              when :tls

data/test/test_config.rb

@@ -344,27 +344,6 @@ class ConfigTest < Test::Unit::TestCase
     File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
   end
 
-  def test_inline
-    prepare_config
-    opts = {
-      :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
-      :use_v1_config => false
-    }
-    assert_nothing_raised do
-      Fluent::Supervisor.new(opts)
-    end
-    create_warn_dummy_logger
-  end
-
-  def create_warn_dummy_logger
-    dl_opts = {}
-    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
-    logdev = Fluent::Test::DummyLogDevice.new
-    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
-    $log = Fluent::Log.new(logger)
-  end
-
   sub_test_case '.build' do
     test 'read config' do
       write_config("#{TMP_DIR}/build/config_build.conf", 'key value')

data/test/test_formatter.rb

@@ -18,7 +18,7 @@ module FormatterTest
 
     def test_call
       formatter = Formatter.new
-      formatter.configure({})
+      formatter.configure(config_element())
       assert_raise NotImplementedError do
         formatter.format('tag', Engine.now, {})
       end
@@ -130,7 +130,7 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::MessagePackFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::MessagePackFormatter)
       @time = Engine.now
     end
 
@@ -146,7 +146,7 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::LabeledTSVFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::LabeledTSVFormatter)
       @time = Engine.now
       @newline = if Fluent.windows?
                    "\r\n"
@@ -156,16 +156,16 @@ module FormatterTest
     end
 
     def test_config_params
-      assert_equal "\t", @formatter.delimiter
-      assert_equal  ":", @formatter.label_delimiter
+      assert_equal "\t", @formatter.instance.delimiter
+      assert_equal  ":", @formatter.instance.label_delimiter
 
       @formatter.configure(
         'delimiter'       => ',',
         'label_delimiter' => '=',
       )
 
-      assert_equal ",", @formatter.delimiter
-      assert_equal "=", @formatter.label_delimiter
+      assert_equal ",", @formatter.instance.delimiter
+      assert_equal "=", @formatter.instance.label_delimiter
     end
 
     def test_format
@@ -220,14 +220,14 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::CsvFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::CsvFormatter)
       @time = Engine.now
     end
 
     def test_config_params
-      assert_equal ',', @formatter.delimiter
-      assert_equal true, @formatter.force_quotes
-      assert_nil @formatter.fields
+      assert_equal ',', @formatter.instance.delimiter
+      assert_equal true, @formatter.instance.force_quotes
+      assert_nil @formatter.instance.fields
     end
 
     data(
@@ -237,8 +237,8 @@ module FormatterTest
     def test_config_params_with_customized_delimiters(data)
       expected, target = data
       @formatter.configure('delimiter' => target, 'fields' => 'a,b,c')
-      assert_equal expected, @formatter.delimiter
-      assert_equal ['a', 'b', 'c'], @formatter.fields
+      assert_equal expected, @formatter.instance.delimiter
+      assert_equal ['a', 'b', 'c'], @formatter.instance.fields
     end
 
     def test_format
@@ -299,7 +299,7 @@ module FormatterTest
       'blank' => 'one,,two,three')
     def test_config_params_with_fields(data)
       @formatter.configure('fields' => data)
-      assert_equal %w(one two three), @formatter.fields
+      assert_equal %w(one two three), @formatter.instance.fields
     end
   end
 
@@ -313,31 +313,34 @@ module FormatterTest
                  end
     end
 
+    def create_driver(klass_or_str)
+      Fluent::Test::FormatterTestDriver.new(klass_or_str)
+    end
 
     def test_config_params
-      formatter = TextFormatter::SingleValueFormatter.new
-      assert_equal "message", formatter.message_key
+      formatter = create_driver(TextFormatter::SingleValueFormatter)
+      assert_equal "message", formatter.instance.message_key
 
       formatter.configure('message_key' => 'foobar')
-      assert_equal "foobar", formatter.message_key
+      assert_equal "foobar", formatter.instance.message_key
     end
 
     def test_format
-      formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter = create_driver('single_value')
       formatter.configure({})
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
       assert_equal("awesome#{@newline}", formatted)
     end
 
     def test_format_without_newline
-      formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter = create_driver('single_value')
       formatter.configure('add_newline' => 'false')
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
       assert_equal("awesome", formatted)
     end
 
     def test_format_with_message_key
-      formatter = TextFormatter::SingleValueFormatter.new
+      formatter = create_driver(TextFormatter::SingleValueFormatter)
       formatter.configure('message_key' => 'foobar')
       formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})