fluentd 1.14.0-x64-mingw32 → 1.14.4-x64-mingw32

data/lib/fluent/plugin/out_forward.rb

@@ -167,6 +167,8 @@ module Fluent::Plugin
       @usock = nil
       @keep_alive_watcher_interval = 5 # TODO
       @suspend_flush = false
+      @healthy_nodes_count_metrics = nil
+      @registered_nodes_count_metrics = nil
     end
 
     def configure(conf)
@@ -265,6 +267,9 @@ module Fluent::Plugin
       end
 
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
+      @healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
+      @registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
+
     end
 
     def multi_workers_ready?
@@ -418,18 +423,21 @@ module Fluent::Plugin
     def statistics
       stats = super
       services = service_discovery_services
-      healthy_nodes_count = 0
-      registed_nodes_count = services.size
+      @healthy_nodes_count_metrics.set(0)
+      @registered_nodes_count_metrics.set(services.size)
       services.each do |s|
         if s.available?
-          healthy_nodes_count += 1
+          @healthy_nodes_count_metrics.inc
         end
       end
 
-      stats.merge(
-        'healthy_nodes_count' => healthy_nodes_count,
-        'registered_nodes_count' => registed_nodes_count,
-      )
+      stats = {
+        'output' => stats["output"].merge({
+          'healthy_nodes_count' => @healthy_nodes_count_metrics.get,
+          'registered_nodes_count' => @registered_nodes_count_metrics.get,
+        })
+      }
+      stats
     end
 
     # MessagePack FixArray length is 3

data/lib/fluent/plugin/output.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/error'
 require 'fluent/plugin/base'
 require 'fluent/plugin/buffer'
@@ -1248,8 +1249,8 @@ module Fluent
           backup_dir = File.dirname(backup_file)
 
           log.warn "bad chunk is moved to #{backup_file}"
-          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(backup_dir)
-          File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
+          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
+          File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
             chunk.write_to(f)
           }
         end
@@ -1275,7 +1276,7 @@ module Fluent
           unless @retry
             @retry = retry_state(@buffer_config.retry_randomize)
             if error
-              log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
               log.warn_backtrace error.backtrace
             end
             return
@@ -1304,11 +1305,11 @@ module Fluent
             if error
               if using_secondary
                 msg = "failed to flush the buffer with secondary output."
-                log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
                 log.warn_backtrace error.backtrace
               else
                 msg = "failed to flush the buffer."
-                log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
                 log.warn_backtrace error.backtrace
               end
             end

data/lib/fluent/plugin/parser_apache2.rb

@@ -21,7 +21,7 @@ module Fluent
     class Apache2Parser < Parser
       Plugin.register_parser('apache2', self)
 
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\")*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\")*)" "(?<agent>(?:[^\"]|\\")*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize

data/lib/fluent/plugin/storage_local.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/plugin'
 require 'fluent/plugin/storage'
 
@@ -25,14 +26,11 @@ module Fluent
     class LocalStorage < Storage
       Fluent::Plugin.register_storage('local', self)
 
-      DEFAULT_DIR_MODE = 0755
-      DEFAULT_FILE_MODE = 0644
-
       config_param :path, :string, default: nil
-      config_param :mode, default: DEFAULT_FILE_MODE do |v|
+      config_param :mode, default: Fluent::DEFAULT_FILE_PERMISSION do |v|
         v.to_i(8)
       end
-      config_param :dir_mode, default: DEFAULT_DIR_MODE do |v|
+      config_param :dir_mode, default: Fluent::DEFAULT_DIR_PERMISSION do |v|
         v.to_i(8)
       end
       config_param :pretty_print, :bool, default: false

data/lib/fluent/plugin_id.rb

@@ -15,6 +15,7 @@
 #
 
 require 'set'
+require 'fluent/env'
 require 'fluent/variable_store'
 
 module Fluent
@@ -76,7 +77,7 @@ module Fluent
 
       # Fluent::Plugin::Base#fluentd_worker_id
       dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
-      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(dir)
+      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
       @_plugin_root_dir = dir.freeze
       dir
     end

data/lib/fluent/supervisor.rb

@@ -544,7 +544,7 @@ module Fluent
         $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
 
         if @path && log_dir_perm
-          File.chmod(log_dir_perm || 0755, File.dirname(@path))
+          File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
         end
       end
 
@@ -651,7 +651,7 @@ module Fluent
           end
         else
           begin
-            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || 0755)
+            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)
           rescue => e
             raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
           end

data/lib/fluent/time.rb

@@ -226,19 +226,16 @@ module Fluent
 
       format_with_timezone = format && (format.include?("%z") || format.include?("%Z"))
 
-      # unixtime_in_expected_tz = unixtime_in_localtime + offset_diff
-      offset_diff = case
-                    when format_with_timezone then nil
-                    when timezone  then
-                      offset = Fluent::Timezone.utc_offset(timezone)
-                      if offset.respond_to?(:call)
-                        ->(t) { Time.now.localtime.utc_offset - offset.call(t) }
-                      else
-                        Time.now.localtime.utc_offset - offset
-                      end
-                    when localtime then 0
-                    else Time.now.localtime.utc_offset # utc
-                    end
+      utc_offset = case
+                   when format_with_timezone then
+                     nil
+                   when timezone then
+                     Fluent::Timezone.utc_offset(timezone)
+                   when localtime then
+                     nil
+                   else
+                     0 # utc
+                   end
 
       strptime = format && (Strptime.new(format) rescue nil)
 
@@ -247,16 +244,20 @@ module Fluent
                when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
                when format == '%iso8601'             then ->(v){ Fluent::EventTime.from_time(Time.iso8601(v)) }
                when strptime then
-                 if offset_diff.respond_to?(:call)
-                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+                 if utc_offset.nil?
+                   ->(v){ t = strptime.exec(v); Fluent::EventTime.new(t.to_i, t.nsec) }
+                 elsif utc_offset.respond_to?(:call)
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }
                  else
-                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }
                  end
-               when format   then
-                 if offset_diff.respond_to?(:call)
-                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+               when format then
+                 if utc_offset.nil?
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i, t.nsec) }
+                 elsif utc_offset.respond_to?(:call)
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }
                  else
-                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }
                  end
                else ->(v){ Fluent::EventTime.parse(v) }
                end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.14.0'
+  VERSION = '1.14.4'
 
 end

data/test/command/test_cat.rb

@@ -69,7 +69,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     def test_cat_json
       d = create_driver
       d.run(expect_records: 1) do
-        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
           stdin.puts('{"key":"value"}')
           stdin.close
         end
@@ -86,7 +86,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       path = d.instance.write(@chunk)
       d = create_driver
       d.run(expect_records: 1) do
-        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
           stdin.write(File.read(path))
           stdin.close
         end
@@ -96,4 +96,33 @@ class TestFluentCat < ::Test::Unit::TestCase
                    [d.events.size, event.first, event.last])
     end
   end
+
+  sub_test_case "send specific event time" do
+    def test_without_event_time
+      event_time = Fluent::EventTime.now
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} tag") do |stdin|
+          stdin.puts('{"key":"value"}')
+          stdin.close
+        end
+      end
+      event = d.events.first
+      assert_in_delta(event_time.to_f, event[1].to_f, 3.0) # expect command to be finished in 3 seconds
+      assert_equal([1, "tag", true, @record],
+                   [d.events.size, event.first, event_time.to_f < event[1].to_f, event.last])
+    end
+
+    def test_with_event_time
+      event_time = "2021-01-02 13:14:15.0+00:00"
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --event-time '#{event_time}' tag") do |stdin|
+          stdin.puts('{"key":"value"}')
+          stdin.close
+        end
+      end
+      assert_equal([["tag", Fluent::EventTime.parse(event_time), @record]], d.events)
+    end
+  end
 end

data/test/plugin/in_tail/test_io_handler.rb

@@ -1,11 +1,19 @@
 require_relative '../../helper'
 
 require 'fluent/plugin/in_tail'
+require 'fluent/plugin/metrics_local'
 require 'tempfile'
 
 class IntailIOHandlerTest < Test::Unit::TestCase
   setup do
     @file = Tempfile.new('intail_io_handler').binmode
+    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+    opened_file_metrics.configure(config_element('metrics', '', {}))
+    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+    closed_file_metrics.configure(config_element('metrics', '', {}))
+    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+    rotated_file_metrics.configure(config_element('metrics', '', {}))
+    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
   end
 
   teardown do
@@ -30,7 +38,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
     end
 
     returned_lines = ''
-    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
       returned_lines << lines.join
       true
     end
@@ -62,7 +70,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = ''
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.join
         true
       end
@@ -93,7 +101,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end
@@ -119,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end

data/test/plugin/test_bare_output.rb

@@ -83,7 +83,7 @@ class BareOutputTest < Test::Unit::TestCase
 
     @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
 
-    assert{ @p.log.object_id != original_logger.object_id }
+    assert(@p.log.object_id != original_logger.object_id)
     assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
   end
 

data/test/plugin/test_buffer.rb

@@ -18,7 +18,7 @@ module FluentPluginBufferTest
   end
   class DummyMemoryChunkError < StandardError; end
   class DummyMemoryChunk < Fluent::Plugin::Buffer::MemoryChunk
-    attr_reader :append_count, :rollbacked, :closed, :purged
+    attr_reader :append_count, :rollbacked, :closed, :purged, :chunk
     attr_accessor :failing
     def initialize(metadata, compress: :text)
       super
@@ -944,6 +944,97 @@ class BufferTest < Test::Unit::TestCase
         @p.write({@dm0 => es}, format: @format)
       end
     end
+
+    data(
+      first_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
+                                                 [event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
+                                                 [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}]]),
+      intermediate_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
+                                                        [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
+                                                        [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}]]),
+      last_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
+                                                [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}],
+                                                [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}]]),
+      multiple_chunks: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
+                                                     [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
+                                                     [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}],
+                                                     [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}]])
+    )
+    test '#write exceeds chunk_limit_size, raise BufferChunkOverflowError, but not lost whole messages' do |(es)|
+      assert_equal [@dm0], @p.stage.keys
+      assert_equal [], @p.queue.map(&:metadata)
+
+      assert_equal 1_280_000, @p.chunk_limit_size
+
+      nth = []
+      es.entries.each_with_index do |entry, index|
+        if entry.last["message"].size == @p.chunk_limit_size
+          nth << index
+        end
+      end
+      messages = []
+      nth.each do |n|
+        messages << "a 1280025 bytes record (nth: #{n}) is larger than buffer chunk limit size (1280000)"
+      end
+
+      assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages.join(", ")) do
+        @p.write({@dm0 => es}, format: @format)
+      end
+      # message a and b are concatenated and staged
+      staged_messages = Fluent::MessagePackFactory.msgpack_unpacker.feed_each(@p.stage[@dm0].chunk).collect do |record|
+        record.last
+      end
+      assert_equal([2, [{"message" => "a"}, {"message" => "b"}]],
+                   [@p.stage[@dm0].size, staged_messages])
+      # only es0 message is queued
+      assert_equal [@dm0], @p.queue.map(&:metadata)
+      assert_equal [5000], @p.queue.map(&:size)
+    end
+
+    test "confirm that every message which is smaller than chunk threshold does not raise BufferChunkOverflowError" do
+      assert_equal [@dm0], @p.stage.keys
+      assert_equal [], @p.queue.map(&:metadata)
+      timestamp = event_time('2016-04-11 16:00:02 +0000')
+      es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "a" * 1_000_000}],
+                                         [timestamp, {"message" => "b" * 1_000_000}],
+                                         [timestamp, {"message" => "c" * 1_000_000}]])
+
+      # https://github.com/fluent/fluentd/issues/1849
+      # Even though 1_000_000 < 1_280_000 (chunk_limit_size), it raised BufferChunkOverflowError before.
+      # It should not be raised and message a,b,c should be stored into 3 chunks.
+      assert_nothing_raised do
+        @p.write({@dm0 => es}, format: @format)
+      end
+      messages = []
+      # pick up first letter to check whether chunk is queued in expected order
+      3.times do |index|
+        chunk = @p.queue[index]
+        es = Fluent::MessagePackEventStream.new(chunk.chunk)
+        es.ensure_unpacked!
+        records = es.instance_eval{ @unpacked_records }
+        records.each do |record|
+          messages << record["message"][0]
+        end
+      end
+      es = Fluent::MessagePackEventStream.new(@p.stage[@dm0].chunk)
+      es.ensure_unpacked!
+      staged_message = es.instance_eval{ @unpacked_records }.first["message"]
+      # message a and b are queued, message c is staged
+      assert_equal([
+                     [@dm0],
+                     "c" * 1_000_000,
+                     [@dm0, @dm0, @dm0],
+                     [5000, 1, 1],
+                     [["x"] * 5000, "a", "b"].flatten
+                   ],
+                   [
+                     @p.stage.keys,
+                     staged_message,
+                     @p.queue.map(&:metadata),
+                     @p.queue.map(&:size),
+                     messages
+                   ])
+    end
   end
 
   sub_test_case 'custom format with configuration for test with lower chunk limit size' do
@@ -1032,6 +1123,38 @@ class BufferTest < Test::Unit::TestCase
         @p.write({@dm0 => es})
       end
     end
+
+    test 'confirm that every array message which is smaller than chunk threshold does not raise BufferChunkOverflowError' do
+      assert_equal [@dm0], @p.stage.keys
+      assert_equal [], @p.queue.map(&:metadata)
+
+      assert_equal 1_280_000, @p.chunk_limit_size
+
+      es = ["a" * 1_000_000, "b" * 1_000_000, "c" * 1_000_000]
+      assert_nothing_raised do
+        @p.write({@dm0 => es})
+      end
+      queue_messages = @p.queue.collect do |chunk|
+        # collect first character of each message
+        chunk.chunk[0]
+      end
+      assert_equal([
+                     [@dm0],
+                     1,
+                     "c",
+                     [@dm0, @dm0, @dm0],
+                     [5000, 1, 1],
+                     ["x", "a", "b"]
+                   ],
+                   [
+                     @p.stage.keys,
+                     @p.stage[@dm0].size,
+                     @p.stage[@dm0].chunk[0],
+                     @p.queue.map(&:metadata),
+                     @p.queue.map(&:size),
+                     queue_messages
+                   ])
+    end
   end
 
   sub_test_case 'with configuration for test with lower limits' do
@@ -1201,6 +1324,7 @@ class BufferTest < Test::Unit::TestCase
   sub_test_case 'when compress is gzip' do
     setup do
       @p = create_buffer({'compress' => 'gzip'})
+      @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
     end
 
     test '#compress returns :gzip' do
@@ -1211,6 +1335,30 @@ class BufferTest < Test::Unit::TestCase
       chunk = @p.generate_chunk(create_metadata)
       assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
     end
+
+    test '#write compressed data which exceeds chunk_limit_size, it raises BufferChunkOverflowError' do
+      @p = create_buffer({'compress' => 'gzip', 'chunk_limit_size' => 70})
+      timestamp = event_time('2016-04-11 16:00:02 +0000')
+      es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "012345"}], # overflow
+                                         [timestamp, {"message" => "aaa"}],
+                                         [timestamp, {"message" => "bbb"}]])
+      assert_equal [], @p.queue.map(&:metadata)
+      assert_equal 70, @p.chunk_limit_size
+
+      # calculate the actual boundary value. it varies on machine
+      c = @p.generate_chunk(create_metadata)
+      c.append(Fluent::ArrayEventStream.new([[timestamp, {"message" => "012345"}]]), compress: :gzip)
+      overflow_bytes = c.bytesize
+
+      messages = "concatenated/appended a #{overflow_bytes} bytes record (nth: 0) is larger than buffer chunk limit size (70)"
+      assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages) do
+        # test format == nil && compress == :gzip
+        @p.write({@dm0 => es})
+      end
+      # message a and b occupies each chunks in full, so both of messages are queued (no staged chunk)
+      assert_equal([2, [@dm0, @dm0], [1, 1], nil],
+                   [@p.queue.size, @p.queue.map(&:metadata), @p.queue.map(&:size), @p.stage[@dm0]])
+    end
   end
 
   sub_test_case '#statistics' do

data/test/plugin/test_filter.rb

@@ -153,7 +153,7 @@ class FilterPluginTest < Test::Unit::TestCase
 
       @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
 
-      assert{ @p.log.object_id != original_logger.object_id }
+      assert(@p.log.object_id != original_logger.object_id)
       assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
     end