fluentd 1.13.3 → 1.16.5

data/test/plugin/test_buf_file_single.rb

@@ -830,4 +830,69 @@ class FileSingleBufferTest < Test::Unit::TestCase
       assert_equal :queued, queue[0].state
     end
   end
+
+  sub_test_case 'there are existing broken file chunks' do
+    setup do
+      FileUtils.rm_rf(@bufdir) rescue nil
+      FileUtils.mkdir_p(@bufdir)
+    end
+
+    teardown do
+      return unless @p
+
+      @p.stop unless @p.stopped?
+      @p.before_shutdown unless @p.before_shutdown?
+      @p.shutdown unless @p.shutdown?
+      @p.after_shutdown unless @p.after_shutdown?
+      @p.close unless @p.closed?
+      @p.terminate unless @p.terminated?
+    end
+
+    test '#resume backups empty chunk' do
+      id_output = 'backup_test'
+      @d = create_driver(%[
+        @id #{id_output}
+        <buffer tag>
+         @type file_single
+         path #{PATH}
+        </buffer>
+      ])
+      @p = @d.instance.buffer
+
+      c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
+      File.open(p1, 'wb') { |f| } # create empty chunk file
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
+      assert { not File.exist?(p1) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
+    end
+
+    test '#resume throws away broken chunk with disable_chunk_backup' do
+      id_output = 'backup_test'
+      @d = create_driver(%[
+        @id #{id_output}
+        <buffer tag>
+         @type file_single
+         path #{PATH}
+         disable_chunk_backup true
+        </buffer>
+      ])
+      @p = @d.instance.buffer
+
+      c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
+      File.open(p1, 'wb') { |f| } # create empty chunk file
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
+      assert { not File.exist?(p1) }
+      assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
+    end
+  end
 end

data/test/plugin/test_buffer.rb

@@ -18,7 +18,7 @@ module FluentPluginBufferTest
   end
   class DummyMemoryChunkError < StandardError; end
   class DummyMemoryChunk < Fluent::Plugin::Buffer::MemoryChunk
-    attr_reader :append_count, :rollbacked, :closed, :purged
+    attr_reader :append_count, :rollbacked, :closed, :purged, :chunk
     attr_accessor :failing
     def initialize(metadata, compress: :text)
       super
@@ -238,8 +238,14 @@ class BufferTest < Test::Unit::TestCase
       assert_nil @p.queue
       assert_nil @p.dequeued
       assert_nil @p.queued_num
-      assert_equal 0, @p.stage_size
-      assert_equal 0, @p.queue_size
+      assert_nil @p.stage_length_metrics
+      assert_nil @p.stage_size_metrics
+      assert_nil @p.queue_length_metrics
+      assert_nil @p.queue_size_metrics
+      assert_nil @p.available_buffer_space_ratios_metrics
+      assert_nil @p.total_queued_size_metrics
+      assert_nil @p.newest_timekey_metrics
+      assert_nil @p.oldest_timekey_metrics
       assert_equal [], @p.timekeys
     end
 
@@ -844,6 +850,116 @@ class BufferTest < Test::Unit::TestCase
     test '#compress returns :text' do
       assert_equal :text, @p.compress
     end
+
+    # https://github.com/fluent/fluentd/issues/3089
+    test "closed chunk should not be committed" do
+      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
+      assert_equal 0.95, @p.chunk_full_threshold
+
+      purge_count = 0
+
+      stub.proxy(@p).generate_chunk(anything) do |chunk|
+        stub.proxy(chunk).purge do |result|
+          purge_count += 1
+          result
+        end
+        stub.proxy(chunk).commit do |result|
+          assert_false(chunk.closed?)
+          result
+        end
+        stub.proxy(chunk).rollback do |result|
+          assert_false(chunk.closed?)
+          result
+        end
+        chunk
+      end
+
+      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
+      small_row = "x" * 1024 * 400
+      big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it does't cause BufferOverFlowError.
+
+      # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`
+      @p.write({m => [small_row] * 40 + [big_row] + ["x"]})
+
+      # Above event strem will be splitted twice by `Buffer#write_step_by_step`
+      #
+      # 1. `write_once`: 42 [events] * 1 [stream]
+      # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
+      # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
+      #
+      # The problematic data is built in the 2nd stage.
+      # In the 2nd stage, 5 streams are packed in a chunk.
+      # ((1024 * 400) [bytes] * 4 [events] * 5 [streams] = 8192000 [bytes] < `chunk_limit_size` (8MB)).
+      # So 3 chunks are used to store all data.
+      # The 1st chunk is already staged by `write_once`.
+      # The 2nd & 3rd chunks are newly created as unstaged.
+      # The 3rd chunk is purged before `ShouldRetry`, it's no problem:
+      #   https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L850
+      # The 2nd chunk is purged in `rescue ShouldRetry`:
+      #   https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L862
+      # It causes the issue described in https://github.com/fluent/fluentd/issues/3089#issuecomment-1811839198
+
+      assert_equal 2, purge_count
+    end
+
+    # https://github.com/fluent/fluentd/issues/4446
+    test "#write_step_by_step keeps chunks kept in locked in entire #write process" do
+      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
+      assert_equal 0.95, @p.chunk_full_threshold
+
+      mon_enter_counts_by_chunk = {}
+      mon_exit_counts_by_chunk = {}
+
+      stub.proxy(@p).generate_chunk(anything) do |chunk|
+        stub(chunk).mon_enter do
+          enter_count = 1 + mon_enter_counts_by_chunk.fetch(chunk, 0)
+          exit_count = mon_exit_counts_by_chunk.fetch(chunk, 0)
+          mon_enter_counts_by_chunk[chunk] = enter_count
+
+          # Assert that chunk is passed to &block of write_step_by_step before exiting the lock.
+          # (i.e. The lock count must be 2 greater than the exit count).
+          # Since ShouldRetry occurs once, the staged chunk takes the lock 3 times when calling the block.
+          if chunk.staged?
+            lock_in_block = enter_count == 3
+            assert_equal(enter_count - 2, exit_count) if lock_in_block
+          else
+            lock_in_block = enter_count == 2
+            assert_equal(enter_count - 2, exit_count) if lock_in_block
+          end
+        end
+        stub(chunk).mon_exit do
+          exit_count = 1 + mon_exit_counts_by_chunk.fetch(chunk, 0)
+          mon_exit_counts_by_chunk[chunk] = exit_count
+        end
+        chunk
+      end
+
+      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
+      small_row = "x" * 1024 * 400
+      big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it does't cause BufferOverFlowError.
+
+      # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`
+      @p.write({m => [small_row] * 40 + [big_row] + ["x"]})
+
+      # Above event strem will be splitted twice by `Buffer#write_step_by_step`
+      #
+      # 1. `write_once`: 42 [events] * 1 [stream]
+      # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
+      # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
+      # 
+      # Example of staged chunk lock behavior:
+      # 
+      # 1. mon_enter in write_step_by_step
+      # 2. ShouldRetry occurs
+      # 3. mon_exit in write_step_by_step
+      # 4. mon_enter again in write_step_by_step (retry)
+      # 5. passed to &block of write_step_by_step
+      # 6. mon_enter in the block (write)
+      # 7. mon_exit in write_step_by_step
+      # 8. mon_exit in write
+
+      assert_equal(mon_enter_counts_by_chunk.values, mon_exit_counts_by_chunk.values)
+    end
   end
 
   sub_test_case 'standard format with configuration for test with lower chunk limit size' do
@@ -938,6 +1054,97 @@ class BufferTest < Test::Unit::TestCase
         @p.write({@dm0 => es}, format: @format)
       end
     end
+
+    data(
+      first_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
+                                                 [event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
+                                                 [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}]]),
+      intermediate_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
+                                                        [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
+                                                        [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}]]),
+      last_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
+                                                [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}],
+                                                [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}]]),
+      multiple_chunks: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
+                                                     [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
+                                                     [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}],
+                                                     [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}]])
+    )
+    test '#write exceeds chunk_limit_size, raise BufferChunkOverflowError, but not lost whole messages' do |(es)|
+      assert_equal [@dm0], @p.stage.keys
+      assert_equal [], @p.queue.map(&:metadata)
+
+      assert_equal 1_280_000, @p.chunk_limit_size
+
+      nth = []
+      es.entries.each_with_index do |entry, index|
+        if entry.last["message"].size == @p.chunk_limit_size
+          nth << index
+        end
+      end
+      messages = []
+      nth.each do |n|
+        messages << "a 1280025 bytes record (nth: #{n}) is larger than buffer chunk limit size (1280000)"
+      end
+
+      assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages.join(", ")) do
+        @p.write({@dm0 => es}, format: @format)
+      end
+      # message a and b are concatenated and staged
+      staged_messages = Fluent::MessagePackFactory.msgpack_unpacker.feed_each(@p.stage[@dm0].chunk).collect do |record|
+        record.last
+      end
+      assert_equal([2, [{"message" => "a"}, {"message" => "b"}]],
+                   [@p.stage[@dm0].size, staged_messages])
+      # only es0 message is queued
+      assert_equal [@dm0], @p.queue.map(&:metadata)
+      assert_equal [5000], @p.queue.map(&:size)
+    end
+
+    test "confirm that every message which is smaller than chunk threshold does not raise BufferChunkOverflowError" do
+      assert_equal [@dm0], @p.stage.keys
+      assert_equal [], @p.queue.map(&:metadata)
+      timestamp = event_time('2016-04-11 16:00:02 +0000')
+      es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "a" * 1_000_000}],
+                                         [timestamp, {"message" => "b" * 1_000_000}],
+                                         [timestamp, {"message" => "c" * 1_000_000}]])
+
+      # https://github.com/fluent/fluentd/issues/1849
+      # Even though 1_000_000 < 1_280_000 (chunk_limit_size), it raised BufferChunkOverflowError before.
+      # It should not be raised and message a,b,c should be stored into 3 chunks.
+      assert_nothing_raised do
+        @p.write({@dm0 => es}, format: @format)
+      end
+      messages = []
+      # pick up first letter to check whether chunk is queued in expected order
+      3.times do |index|
+        chunk = @p.queue[index]
+        es = Fluent::MessagePackEventStream.new(chunk.chunk)
+        es.ensure_unpacked!
+        records = es.instance_eval{ @unpacked_records }
+        records.each do |record|
+          messages << record["message"][0]
+        end
+      end
+      es = Fluent::MessagePackEventStream.new(@p.stage[@dm0].chunk)
+      es.ensure_unpacked!
+      staged_message = es.instance_eval{ @unpacked_records }.first["message"]
+      # message a and b are queued, message c is staged
+      assert_equal([
+                     [@dm0],
+                     "c" * 1_000_000,
+                     [@dm0, @dm0, @dm0],
+                     [5000, 1, 1],
+                     [["x"] * 5000, "a", "b"].flatten
+                   ],
+                   [
+                     @p.stage.keys,
+                     staged_message,
+                     @p.queue.map(&:metadata),
+                     @p.queue.map(&:size),
+                     messages
+                   ])
+    end
   end
 
   sub_test_case 'custom format with configuration for test with lower chunk limit size' do
@@ -1026,6 +1233,38 @@ class BufferTest < Test::Unit::TestCase
         @p.write({@dm0 => es})
       end
     end
+
+    test 'confirm that every array message which is smaller than chunk threshold does not raise BufferChunkOverflowError' do
+      assert_equal [@dm0], @p.stage.keys
+      assert_equal [], @p.queue.map(&:metadata)
+
+      assert_equal 1_280_000, @p.chunk_limit_size
+
+      es = ["a" * 1_000_000, "b" * 1_000_000, "c" * 1_000_000]
+      assert_nothing_raised do
+        @p.write({@dm0 => es})
+      end
+      queue_messages = @p.queue.collect do |chunk|
+        # collect first character of each message
+        chunk.chunk[0]
+      end
+      assert_equal([
+                     [@dm0],
+                     1,
+                     "c",
+                     [@dm0, @dm0, @dm0],
+                     [5000, 1, 1],
+                     ["x", "a", "b"]
+                   ],
+                   [
+                     @p.stage.keys,
+                     @p.stage[@dm0].size,
+                     @p.stage[@dm0].chunk[0],
+                     @p.queue.map(&:metadata),
+                     @p.queue.map(&:size),
+                     queue_messages
+                   ])
+    end
   end
 
   sub_test_case 'with configuration for test with lower limits' do
@@ -1195,6 +1434,7 @@ class BufferTest < Test::Unit::TestCase
   sub_test_case 'when compress is gzip' do
     setup do
       @p = create_buffer({'compress' => 'gzip'})
+      @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
     end
 
     test '#compress returns :gzip' do
@@ -1205,6 +1445,30 @@ class BufferTest < Test::Unit::TestCase
       chunk = @p.generate_chunk(create_metadata)
       assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
     end
+
+    test '#write compressed data which exceeds chunk_limit_size, it raises BufferChunkOverflowError' do
+      @p = create_buffer({'compress' => 'gzip', 'chunk_limit_size' => 70})
+      timestamp = event_time('2016-04-11 16:00:02 +0000')
+      es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "012345"}], # overflow
+                                         [timestamp, {"message" => "aaa"}],
+                                         [timestamp, {"message" => "bbb"}]])
+      assert_equal [], @p.queue.map(&:metadata)
+      assert_equal 70, @p.chunk_limit_size
+
+      # calculate the actual boundary value. it varies on machine
+      c = @p.generate_chunk(create_metadata)
+      c.append(Fluent::ArrayEventStream.new([[timestamp, {"message" => "012345"}]]), compress: :gzip)
+      overflow_bytes = c.bytesize
+
+      messages = "concatenated/appended a #{overflow_bytes} bytes record (nth: 0) is larger than buffer chunk limit size (70)"
+      assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages) do
+        # test format == nil && compress == :gzip
+        @p.write({@dm0 => es})
+      end
+      # message a and b occupies each chunks in full, so both of messages are queued (no staged chunk)
+      assert_equal([2, [@dm0, @dm0], [1, 1], nil],
+                   [@p.queue.size, @p.queue.map(&:metadata), @p.queue.map(&:size), @p.stage[@dm0]])
+    end
   end
 
   sub_test_case '#statistics' do

data/test/plugin/test_buffer_chunk.rb

@@ -57,6 +57,17 @@ class BufferChunkTest < Test::Unit::TestCase
       assert chunk.respond_to?(:msgpack_each)
     end
 
+    test 'unpacker arg is not implemented for ChunkMessagePackEventStreamer' do
+      meta = Object.new
+      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
+      chunk.extend Fluent::ChunkMessagePackEventStreamer
+
+      unpacker = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
+
+      assert_raise(NotImplementedError){ chunk.each(unpacker: unpacker) }
+      assert_raise(NotImplementedError){ chunk.msgpack_each(unpacker: unpacker) }
+    end
+
     test 'some methods raise ArgumentError with an option of `compressed: :gzip` and without extending Compressble`' do
       meta = Object.new
       chunk = Fluent::Plugin::Buffer::Chunk.new(meta)

data/test/plugin/test_filter.rb

@@ -153,7 +153,7 @@ class FilterPluginTest < Test::Unit::TestCase
 
       @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
 
-      assert{ @p.log.object_id != original_logger.object_id }
+      assert(@p.log.object_id != original_logger.object_id)
       assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
     end
 
@@ -165,6 +165,17 @@ class FilterPluginTest < Test::Unit::TestCase
       end
     end
 
+    test 'can use metrics plugins and fallback methods' do
+      @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+      %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+        assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @p.emit_size
+      assert_equal 0, @p.emit_records
+    end
+
     test 'are available with multi worker configuration in default' do
       assert @p.multi_workers_ready?
     end

data/test/plugin/test_filter_parser.rb

@@ -46,7 +46,7 @@ class ParserFilterTest < Test::Unit::TestCase
     assert_raise(Fluent::ConfigError) {
       create_driver('')
     }
-    assert_raise(Fluent::ConfigError) {
+    assert_raise(Fluent::NotFoundPluginError) {
       create_driver %[
         key_name foo
         <parse>

data/test/plugin/test_filter_stdout.rb

@@ -63,7 +63,7 @@ class StdoutFilterTest < Test::Unit::TestCase
       end
 
       def test_invalid_output_type
-        assert_raise(Fluent::ConfigError) do
+        assert_raise(Fluent::NotFoundPluginError) do
           d = create_driver(CONFIG + config_element("", "", { "output_type" => "foo" }))
           d.run {}
         end
@@ -139,7 +139,7 @@ class StdoutFilterTest < Test::Unit::TestCase
       def test_invalid_output_type
         conf = config_element
         conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "foo" })
-        assert_raise(Fluent::ConfigError) do
+        assert_raise(Fluent::NotFoundPluginError) do
           d = create_driver(conf)
           d.run {}
         end

data/test/plugin/test_in_forward.rb

@@ -68,7 +68,6 @@ class ForwardInputTest < Test::Unit::TestCase
       @d = d = create_driver
       assert_equal @port, d.instance.port
       assert_equal '127.0.0.1', d.instance.bind
-      assert_equal 0, d.instance.linger_timeout
       assert_equal 0.5, d.instance.blocking_timeout
       assert !d.instance.backlog
     end
@@ -77,7 +76,6 @@ class ForwardInputTest < Test::Unit::TestCase
       @d = d = create_driver(config_auth)
       assert_equal @port, d.instance.port
       assert_equal '127.0.0.1', d.instance.bind
-      assert_equal 0, d.instance.linger_timeout
       assert !d.instance.backlog
 
       assert d.instance.security
@@ -369,7 +367,7 @@ class ForwardInputTest < Test::Unit::TestCase
       end
 
       logs = d.instance.log.out.logs
-      assert{ logs.select{|line| line =~ /skip invalid event/ }.size == 2 }
+      assert{ logs.count{|line| line =~ /skip invalid event/ } == 2 }
 
       d.instance_shutdown
     end
@@ -595,10 +593,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -621,10 +619,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{ |line|
+      assert_equal 1, logs.count{ |line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -655,10 +653,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=33554432 size=33554989/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -678,9 +676,9 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: incoming chunk is broken: host="#{LOCALHOST_HOSTNAME}" msg=#{data.inspect}/
-      }.size, "should not accept broken chunk"
+      }, "should not accept broken chunk"
 
       d.instance_shutdown
     end

data/test/plugin/test_in_http.rb

@@ -7,9 +7,8 @@ require 'timecop'
 class HttpInputTest < Test::Unit::TestCase
   class << self
     def startup
-      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
-      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      @server = ServerEngine::SocketManager::Server.open
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
     end
 
     def shutdown
@@ -540,6 +539,29 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_application_ndjson
+    d = create_driver
+    events = [
+        ["tag1", 1643935663, "{\"a\":1}\n{\"b\":2}"],
+        ["tag2", 1643935664, "{\"a\":3}\r\n{\"b\":4}"]
+    ]
+
+    expected = [
+        ["tag1", 1643935663, {"a"=>1}],
+        ["tag1", 1643935663, {"b"=>2}],
+        ["tag2", 1643935664, {"a"=>3}],
+        ["tag2", 1643935664, {"b"=>4}]
+    ]
+
+    d.run(expect_records: 1) do
+      events.each do |tag, time, record|
+        res = post("/#{tag}?time=#{time}", record, {"Content-Type"=>"application/x-ndjson"})
+        assert_equal("200", res.code)
+      end
+    end
+    assert_equal(expected, d.events)
+  end
+
   def test_msgpack
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -856,6 +878,46 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_cors_allow_credentials
+    d = create_driver(config + %[
+          cors_allow_origins ["http://foo.com"]
+          cors_allow_credentials
+        ])
+    assert_equal true, d.instance.cors_allow_credentials
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    event = ["tag1", time, {"a"=>1}]
+    res_code = nil
+    res_header = nil
+
+    d.run do
+      res = post("/#{event[0]}", {"json"=>event[2].to_json, "time"=>time.to_i.to_s}, {"Origin"=>"http://foo.com"})
+      res_code = res.code
+      res_header = res["Access-Control-Allow-Credentials"]
+    end
+    assert_equal(
+      {
+        response_code: "200",
+        allow_credentials_header: "true",
+        events: [event]
+      },
+      {
+        response_code: res_code,
+        allow_credentials_header: res_header,
+        events: d.events
+      }
+    )
+  end
+
+  def test_cors_allow_credentials_for_wildcard_origins
+    assert_raise(Fluent::ConfigError) do
+      create_driver(config + %[
+        cors_allow_origins ["*"]
+        cors_allow_credentials
+      ])
+    end
+  end
+
   def test_content_encoding_gzip
     d = create_driver