fluentd 1.15.3-x64-mingw32 → 1.16.1-x64-mingw32

data/test/plugin/test_buf_file.rb

@@ -1151,15 +1151,13 @@ class FileBufferTest < Test::Unit::TestCase
 
   sub_test_case 'there are existing broken file chunks' do
     setup do
+      @id_output = 'backup_test'
       @bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)
-      FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)
+      FileUtils.rm_rf @bufdir rescue nil
+      FileUtils.mkdir_p @bufdir
       @bufpath = File.join(@bufdir, 'broken_test.*.log')
 
       Fluent::Test.setup
-      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
-      @p = Fluent::Plugin::FileBuffer.new
-      @p.owner = @d
-      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
     end
 
     teardown do
@@ -1171,12 +1169,12 @@ class FileBufferTest < Test::Unit::TestCase
         @p.close unless @p.closed?
         @p.terminate unless @p.terminated?
       end
-      if @bufdir
-        Dir.glob(File.join(@bufdir, '*')).each do |path|
-          next if ['.', '..'].include?(File.basename(path))
-          File.delete(path)
-        end
-      end
+    end
+
+    def setup_plugins(buf_conf)
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @d.configure(config_element('ROOT', '', {'@id' => @id_output}, [config_element('buffer', '', buf_conf)]))
+      @p = @d.buffer
     end
 
     def create_first_chunk(mode)
@@ -1232,44 +1230,85 @@ class FileBufferTest < Test::Unit::TestCase
       assert { logs.any? { |log| log.include?(msg) } }
     end
 
-    test '#resume ignores staged empty chunk' do
-      _, p1 = create_first_chunk('b')
+    test '#resume backups staged empty chunk' do
+      setup_plugins({'path' => @bufpath})
+      c1id, p1 = create_first_chunk('b')
       File.open(p1, 'wb') { |f| } # create staged empty chunk file
       c2id, _ = create_second_chunk('b')
 
-      @p.start
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
       compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)
       compare_log(@p, 'staged file chunk is empty')
+      assert { not File.exist?(p1) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
     end
 
-    test '#resume ignores staged broken metadata' do
+    test '#resume backups staged broken metadata' do
+      setup_plugins({'path' => @bufpath})
       c1id, _ = create_first_chunk('b')
-      _, p2 = create_second_chunk('b')
+      c2id, p2 = create_second_chunk('b')
       File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
 
-      @p.start
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
       compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
       compare_log(@p, 'staged meta file is broken')
+      assert { not File.exist?(p2) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
     end
 
-    test '#resume ignores enqueued empty chunk' do
-      _, p1 = create_first_chunk('q')
+    test '#resume backups enqueued empty chunk' do
+      setup_plugins({'path' => @bufpath})
+      c1id, p1 = create_first_chunk('q')
       File.open(p1, 'wb') { |f| } # create enqueued empty chunk file
       c2id, _ = create_second_chunk('q')
 
-      @p.start
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
       compare_queued_chunk(@p.queue, c2id, 3, :queued)
       compare_log(@p, 'enqueued file chunk is empty')
+      assert { not File.exist?(p1) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
     end
 
-    test '#resume ignores enqueued broken metadata' do
+    test '#resume backups enqueued broken metadata' do
+      setup_plugins({'path' => @bufpath})
       c1id, _ = create_first_chunk('q')
-      _, p2 = create_second_chunk('q')
+      c2id, p2 = create_second_chunk('q')
       File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file
 
-      @p.start
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
       compare_queued_chunk(@p.queue, c1id, 4, :queued)
       compare_log(@p, 'enqueued meta file is broken')
+      assert { not File.exist?(p2) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
+    end
+
+    test '#resume throws away broken chunk with disable_chunk_backup' do
+      setup_plugins({'path' => @bufpath, 'disable_chunk_backup' => true})
+      c1id, _ = create_first_chunk('b')
+      c2id, p2 = create_second_chunk('b')
+      File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
+      compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
+      compare_log(@p, 'staged meta file is broken')
+      compare_log(@p, 'disable_chunk_backup is true')
+      assert { not File.exist?(p2) }
+      assert { not File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
     end
   end
 end

data/test/plugin/test_buf_file_single.rb

@@ -830,4 +830,69 @@ class FileSingleBufferTest < Test::Unit::TestCase
       assert_equal :queued, queue[0].state
     end
   end
+
+  sub_test_case 'there are existing broken file chunks' do
+    setup do
+      FileUtils.rm_rf(@bufdir) rescue nil
+      FileUtils.mkdir_p(@bufdir)
+    end
+
+    teardown do
+      return unless @p
+
+      @p.stop unless @p.stopped?
+      @p.before_shutdown unless @p.before_shutdown?
+      @p.shutdown unless @p.shutdown?
+      @p.after_shutdown unless @p.after_shutdown?
+      @p.close unless @p.closed?
+      @p.terminate unless @p.terminated?
+    end
+
+    test '#resume backups empty chunk' do
+      id_output = 'backup_test'
+      @d = create_driver(%[
+        @id #{id_output}
+        <buffer tag>
+         @type file_single
+         path #{PATH}
+        </buffer>
+      ])
+      @p = @d.instance.buffer
+
+      c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
+      File.open(p1, 'wb') { |f| } # create empty chunk file
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
+      assert { not File.exist?(p1) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
+    end
+
+    test '#resume throws away broken chunk with disable_chunk_backup' do
+      id_output = 'backup_test'
+      @d = create_driver(%[
+        @id #{id_output}
+        <buffer tag>
+         @type file_single
+         path #{PATH}
+         disable_chunk_backup true
+        </buffer>
+      ])
+      @p = @d.instance.buffer
+
+      c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
+      File.open(p1, 'wb') { |f| } # create empty chunk file
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
+      assert { not File.exist?(p1) }
+      assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
+    end
+  end
 end

data/test/plugin/test_in_http.rb

@@ -7,9 +7,8 @@ require 'timecop'
 class HttpInputTest < Test::Unit::TestCase
   class << self
     def startup
-      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
-      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      @server = ServerEngine::SocketManager::Server.open
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
     end
 
     def shutdown

data/test/plugin/test_in_monitor_agent.rb

@@ -355,14 +355,13 @@ EOC
 
     test "fluentd opts" do
       d = create_driver
-      opts = Fluent::Supervisor.default_options
 
       filepath = nil
       begin
         FileUtils.mkdir_p(CONFIG_DIR)
         filepath = File.expand_path('fluentd.conf', CONFIG_DIR)
         FileUtils.touch(filepath)
-        s = Fluent::Supervisor.new(opts.merge(config_path: filepath))
+        s = Fluent::Supervisor.new({config_path: filepath})
         s.configure
       ensure
         FileUtils.rm_r(CONFIG_DIR) rescue _
@@ -502,7 +501,7 @@ EOC
           v.puts(conf)
         end
 
-        @supervisor = Fluent::Supervisor.new(Fluent::Supervisor.default_options.merge(config_path: @filepath))
+        @supervisor = Fluent::Supervisor.new({config_path: @filepath})
         @supervisor.configure
       ensure
         FileUtils.rm_r(CONFIG_DIR) rescue _

data/test/plugin/test_in_tcp.rb

@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal hostname, event[2]['host']
   end
 
+  test "send_keepalive_packet_can_be_enabled" do
+    d = create_driver(base_config + %!
+      format none
+      send_keepalive_packet true
+    !)
+    assert_true d.instance.send_keepalive_packet
+
+    d = create_driver(base_config + %!
+      format none
+    !)
+    assert_false d.instance.send_keepalive_packet
+  end
+
   test 'source_address_key' do
     d = create_driver(base_config + %!
       format none
@@ -205,13 +218,13 @@ class TcpInputTest < Test::Unit::TestCase
           </client>
         </security>
       !)
-      d.run(shutdown: false, expect_records: 1, timeout: 2) do
+      d.run(expect_records: 1, timeout: 2) do
         create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send("hello\n", 0)
         end
       end
 
-      assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
+      assert_equal 1, d.logs.count { |l| l =~ /anonymous client/ }
       assert_equal 0, d.events.size
     end
   end
@@ -240,4 +253,76 @@ class TcpInputTest < Test::Unit::TestCase
       assert_equal 'hello', event[2]['msg']
     end
   end
+
+  sub_test_case "message_length_limit" do
+    data("batch_emit", { extract: "" }, keep: true)
+    data("single_emit", { extract: "<extract>\ntag_key tag\n</extract>\n" }, keep: true)
+    test "drop records exceeding limit" do |data|
+      message_length_limit = 10
+      d = create_driver(base_config + %!
+        message_length_limit #{message_length_limit}
+        <parse>
+          @type none
+        </parse>
+        #{data[:extract]}
+      !)
+      d.run(expect_records: 2, timeout: 10) do
+        create_tcp_socket('127.0.0.1', @port) do |sock|
+          sock.send("a" * message_length_limit + "\n", 0)
+          sock.send("b" * (message_length_limit + 1) + "\n", 0)
+          sock.send("c" * (message_length_limit - 1) + "\n", 0)
+        end
+      end
+
+      expected_records = [
+        "a" * message_length_limit,
+        "c" * (message_length_limit - 1)
+      ]
+      actual_records = d.events.collect do |event|
+        event[2]["message"]
+      end
+
+      assert_equal expected_records, actual_records
+    end
+
+    test "clear buffer and discard the subsequent data until the next delimiter" do |data|
+      message_length_limit = 12
+      d = create_driver(base_config + %!
+        message_length_limit #{message_length_limit}
+        delimiter ";"
+        <parse>
+          @type json
+        </parse>
+        #{data[:extract]}
+      !)
+      d.run(expect_records: 1, timeout: 10) do
+        create_tcp_socket('127.0.0.1', @port) do |sock|
+          sock.send('{"message":', 0)
+          sock.send('"hello', 0)
+          sleep 1 # To make the server read data and clear the buffer here.
+          sock.send('world!"};', 0) # This subsequent data must be discarded so that a parsing failure doesn't occur.
+          sock.send('{"k":"v"};', 0) # This will succeed to parse.
+        end
+      end
+
+      logs = d.logs.collect do |log|
+        log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
+      end
+      actual_records = d.events.collect do |event|
+        event[2]
+      end
+
+      assert_equal(
+        {
+          # Asserting that '[warn]: pattern not matched message="world!\"}"' warning does not occur.
+          logs: ['[info]: The buffer size exceeds \'message_length_limit\', cleared: limit=12 size=17 head="{\"message\":\"hello"' + "\n"],
+          records: [{"k" => "v"}],
+        },
+        {
+          logs: logs[1..],
+          records: actual_records,
+        }
+      )
+    end
+  end
 end

data/test/plugin/test_in_udp.rb

@@ -265,4 +265,32 @@ class UdpInputTest < Test::Unit::TestCase
       end
     end
   end
+
+  test 'message_length_limit' do
+    message_length_limit = 32
+    d = create_driver(base_config + %!
+      format none
+      message_length_limit #{message_length_limit}
+    !)
+    d.run(expect_records: 3) do
+      create_udp_socket('127.0.0.1', @port) do |u|
+        3.times do |i|
+          u.send("#{i}" * 40 + "\n", 0)
+        end
+      end
+    end
+
+    if Fluent.windows?
+      expected_records = []
+    else
+      expected_records = 3.times.collect do |i|
+        "#{i}" * message_length_limit
+      end
+    end
+    actual_records = d.events.collect do |event|
+      event[2]["message"]
+    end
+
+    assert_equal expected_records, actual_records
+  end
 end

data/test/plugin/test_out_forward.rb

@@ -1331,26 +1331,22 @@ EOL
         d = create_driver(output_conf)
         d.instance_start
 
-        begin
-          chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-          mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                                   linger_timeout: anything,
-                                                   send_timeout: anything,
-                                                   recv_timeout: anything,
-                                                   connect_timeout: anything) { |sock|
-            mock(sock).close.once; sock
-          }.twice
-
-          target_input_driver.run(timeout: 15) do
-            d.run(shutdown: false) do
-              node = d.instance.nodes.first
-              2.times do
-                node.send_data('test', chunk) rescue nil
-              end
+        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+        mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                                 linger_timeout: anything,
+                                                 send_timeout: anything,
+                                                 recv_timeout: anything,
+                                                 connect_timeout: anything) { |sock|
+          mock(sock).close.once; sock
+        }.twice
+
+        target_input_driver.run(timeout: 15) do
+          d.run do
+            node = d.instance.nodes.first
+            2.times do
+              node.send_data('test', chunk) rescue nil
             end
           end
-        ensure
-          d.instance_shutdown
         end
       end
     end

data/test/plugin/test_out_http.rb

@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
           password hello?
         </auth>
       ])
+      d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
       d.run(default_tag: 'test.http', shutdown: false) do
         test_events.each { |event|
           d.feed(event)