fluentd 1.14.3 → 1.14.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4863e324dc88cd23cff5f68feeb71de9cda4a0c3c4594a515e4545d2bd589105
-  data.tar.gz: 2b791da9f6c98d61199c2361cd6623ea01dcaff7d1e3e93aea8d98fda86df4cb
+  metadata.gz: deabdaa85e1adbc3d54e342e2faefaa2c952b492de3167e68fc4be021c605acb
+  data.tar.gz: f1e08a07176309930295b14c45bebaca86e7a7fd8de1191440d987170c752b96
 SHA512:
-  metadata.gz: d9b7fb44d1801b19578bd1b6ef1c44c57a9027535909669c8bd97d19536d63157a8feef203891f257532a92a41e70f5358611fd10be2a94459f38959fd98716f
-  data.tar.gz: ef4958c9827c8e093c73e46a0fbf33368dcc6ccec156ed0ed8fe4b76ec91f145e53eccb263ed52996e905326bce4362d600c2665a718c047b304cb3a0ddae58e
+  metadata.gz: 2f8d7f216549cfd57266bec6a06a42e2def597073aa103bde30bb2bad816f3a25b6b5b942fb7320d69f15164b887ab951e15039f62e1808162e554d11f1c7db6
+  data.tar.gz: f6a31183c931411e1cd4fae6d3dd8691f06ddfdfed4597bba001415052de1bf0081f1da6c75cf52ff4fd7479bf27894306d6336eb0f320c7adf1451dbcdf579a

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+# v1.14.4
+
+## Release v1.14.4 - 2022/01/06
+
+### Enhancement
+
+* `in_tail`: Add option to skip long lines (`max_line_size`)
+  https://github.com/fluent/fluentd/pull/3565
+
+### Bug fix
+
+* Incorrect BufferChunkOverflowError when each event size is < `chunk_limit_size`
+  https://github.com/fluent/fluentd/pull/3560
+* On macOS with Ruby 2.7/3.0, `out_file` fails to write events if `append` is true.
+  https://github.com/fluent/fluentd/pull/3579
+* test: Fix unstable test cases
+  https://github.com/fluent/fluentd/pull/3574
+  https://github.com/fluent/fluentd/pull/3577
+
 # v1.14.3
 
 ## Release v1.14.3 - 2021/11/26

data/lib/fluent/env.rb

@@ -33,4 +33,8 @@ module Fluent
   def self.linux?
     /linux/ === RUBY_PLATFORM
   end
+
+  def self.macos?
+    /darwin/ =~ RUBY_PLATFORM
+  end
 end

data/lib/fluent/plugin/buffer.rb

@@ -767,24 +767,37 @@ module Fluent
             raise ShouldRetry unless chunk.writable?
             staged_chunk_used = true if chunk.staged?
 
-            original_bytesize = chunk.bytesize
+            original_bytesize = committed_bytesize = chunk.bytesize
             begin
               while writing_splits_index < splits.size
                 split = splits[writing_splits_index]
-                formatted_split = format ? format.call(split) : split.first
-                if split.size == 1 && original_bytesize == 0
-                  if format == nil && @compress != :text
-                    # The actual size of chunk is not determined until after chunk.append.
-                    # so, keep already processed 'split' content here.
-                    # (allow performance regression a bit)
+                formatted_split = format ? format.call(split) : nil
+
+                if split.size == 1 # Check BufferChunkOverflowError
+                  determined_bytesize = nil
+                  if @compress != :text
+                    determined_bytesize = nil
+                  elsif formatted_split
+                    determined_bytesize = formatted_split.bytesize
+                  elsif split.first.respond_to?(:bytesize)
+                    determined_bytesize = split.first.bytesize
+                  end
+
+                  if determined_bytesize && determined_bytesize > @chunk_limit_size
+                    # It is a obvious case that BufferChunkOverflowError should be raised here.
+                    # But if it raises here, already processed 'split' or
+                    # the proceeding 'split' will be lost completely.
+                    # So it is a last resort to delay raising such a exception
+                    errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                    writing_splits_index += 1
+                    next
+                  end
+
+                  if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
+                    # The split will (might) cause size over so keep already processed
+                    # 'split' content here (allow performance regression a bit).
                     chunk.commit
-                  else
-                    big_record_size = formatted_split.bytesize
-                    if chunk.bytesize + big_record_size > @chunk_limit_size
-                      errors << "a #{big_record_size} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                      writing_splits_index += 1
-                      next
-                    end
+                    committed_bytesize = chunk.bytesize
                   end
                 end
 
@@ -793,19 +806,26 @@ module Fluent
                 else
                   chunk.append(split, compress: @compress)
                 end
+                adding_bytes = chunk.bytesize - committed_bytesize
 
                 if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
-                  adding_bytes = chunk.instance_eval { @adding_bytes } || "N/A" # 3rd party might not have 'adding_bytes'
                   chunk.rollback
+                  committed_bytesize = chunk.bytesize
 
-                  if split.size == 1 && original_bytesize == 0
-                    # It is obviously case that BufferChunkOverflowError should be raised here,
-                    # but if it raises here, already processed 'split' or
-                    # the proceeding 'split' will be lost completely.
-                    # so it is a last resort to delay raising such a exception
-                    errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                    writing_splits_index += 1
-                    next
+                  if split.size == 1 # Check BufferChunkOverflowError again
+                    if adding_bytes > @chunk_limit_size
+                      errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                      writing_splits_index += 1
+                      next
+                    else
+                      # As already processed content is kept after rollback, then unstaged chunk should be queued.
+                      # After that, re-process current split again.
+                      # New chunk should be allocated, to do it, modify @stage and so on.
+                      synchronize { @stage.delete(modified_metadata) }
+                      staged_chunk_used = false
+                      chunk.unstaged!
+                      break
+                    end
                   end
 
                   if chunk_size_full?(chunk) || split.size == 1

data/lib/fluent/plugin/in_tail.rb

@@ -113,6 +113,8 @@ module Fluent::Plugin
     config_param :path_timezone, :string, default: nil
     desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'
     config_param :follow_inodes, :bool, default: false
+    desc 'Maximum length of line. The longer line is just skipped.'
+    config_param :max_line_size, :size, default: nil
 
     config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
       config_argument :usage, :string, default: 'in_tail_parser'
@@ -594,6 +596,14 @@ module Fluent::Plugin
 
     # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
     def receive_lines(lines, tail_watcher)
+      lines = lines.reject do |line|
+        skip_line = @max_line_size ? line.bytesize > @max_line_size : false
+        if skip_line
+          log.warn "received line length is longer than #{@max_line_size}"
+          log.debug "skipped line: #{line.chomp}"
+        end
+        skip_line
+      end
       es = @receive_handler.call(lines, tail_watcher)
       unless es.empty?
         tag = if @tag_prefix || @tag_suffix

data/lib/fluent/plugin/out_file.rb

@@ -181,6 +181,13 @@ module Fluent::Plugin
       @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
       @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
       @need_lock = system_config.workers > 1
+
+      # https://github.com/fluent/fluentd/issues/3569
+      @need_ruby_on_macos_workaround = false
+      if @append && Fluent.macos?
+        condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
+        @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
+      end
     end
 
     def multi_workers_ready?
@@ -223,7 +230,12 @@ module Fluent::Plugin
 
     def write_without_compression(path, chunk)
       File.open(path, "ab", @file_perm) do |f|
-        chunk.write_to(f)
+        if @need_ruby_on_macos_workaround
+          content = chunk.read()
+          f.puts content
+        else
+          chunk.write_to(f)
+        end
       end
     end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.14.3'
+  VERSION = '1.14.4'
 
 end

data/test/plugin/test_bare_output.rb

@@ -83,7 +83,7 @@ class BareOutputTest < Test::Unit::TestCase
 
     @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
 
-    assert{ @p.log.object_id != original_logger.object_id }
+    assert(@p.log.object_id != original_logger.object_id)
     assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
   end
 

data/test/plugin/test_buffer.rb

@@ -990,6 +990,51 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm0], @p.queue.map(&:metadata)
       assert_equal [5000], @p.queue.map(&:size)
     end
+
+    test "confirm that every message which is smaller than chunk threshold does not raise BufferChunkOverflowError" do
+      assert_equal [@dm0], @p.stage.keys
+      assert_equal [], @p.queue.map(&:metadata)
+      timestamp = event_time('2016-04-11 16:00:02 +0000')
+      es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "a" * 1_000_000}],
+                                         [timestamp, {"message" => "b" * 1_000_000}],
+                                         [timestamp, {"message" => "c" * 1_000_000}]])
+
+      # https://github.com/fluent/fluentd/issues/1849
+      # Even though 1_000_000 < 1_280_000 (chunk_limit_size), it raised BufferChunkOverflowError before.
+      # It should not be raised and message a,b,c should be stored into 3 chunks.
+      assert_nothing_raised do
+        @p.write({@dm0 => es}, format: @format)
+      end
+      messages = []
+      # pick up first letter to check whether chunk is queued in expected order
+      3.times do |index|
+        chunk = @p.queue[index]
+        es = Fluent::MessagePackEventStream.new(chunk.chunk)
+        es.ensure_unpacked!
+        records = es.instance_eval{ @unpacked_records }
+        records.each do |record|
+          messages << record["message"][0]
+        end
+      end
+      es = Fluent::MessagePackEventStream.new(@p.stage[@dm0].chunk)
+      es.ensure_unpacked!
+      staged_message = es.instance_eval{ @unpacked_records }.first["message"]
+      # message a and b are queued, message c is staged
+      assert_equal([
+                     [@dm0],
+                     "c" * 1_000_000,
+                     [@dm0, @dm0, @dm0],
+                     [5000, 1, 1],
+                     [["x"] * 5000, "a", "b"].flatten
+                   ],
+                   [
+                     @p.stage.keys,
+                     staged_message,
+                     @p.queue.map(&:metadata),
+                     @p.queue.map(&:size),
+                     messages
+                   ])
+    end
   end
 
   sub_test_case 'custom format with configuration for test with lower chunk limit size' do
@@ -1078,6 +1123,38 @@ class BufferTest < Test::Unit::TestCase
         @p.write({@dm0 => es})
       end
     end
+
+    test 'confirm that every array message which is smaller than chunk threshold does not raise BufferChunkOverflowError' do
+      assert_equal [@dm0], @p.stage.keys
+      assert_equal [], @p.queue.map(&:metadata)
+
+      assert_equal 1_280_000, @p.chunk_limit_size
+
+      es = ["a" * 1_000_000, "b" * 1_000_000, "c" * 1_000_000]
+      assert_nothing_raised do
+        @p.write({@dm0 => es})
+      end
+      queue_messages = @p.queue.collect do |chunk|
+        # collect first character of each message
+        chunk.chunk[0]
+      end
+      assert_equal([
+                     [@dm0],
+                     1,
+                     "c",
+                     [@dm0, @dm0, @dm0],
+                     [5000, 1, 1],
+                     ["x", "a", "b"]
+                   ],
+                   [
+                     @p.stage.keys,
+                     @p.stage[@dm0].size,
+                     @p.stage[@dm0].chunk[0],
+                     @p.queue.map(&:metadata),
+                     @p.queue.map(&:size),
+                     queue_messages
+                   ])
+    end
   end
 
   sub_test_case 'with configuration for test with lower limits' do

data/test/plugin/test_filter.rb

@@ -153,7 +153,7 @@ class FilterPluginTest < Test::Unit::TestCase
 
       @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
 
-      assert{ @p.log.object_id != original_logger.object_id }
+      assert(@p.log.object_id != original_logger.object_id)
       assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
     end
 

data/test/plugin/test_in_tail.rb

@@ -1707,6 +1707,41 @@ class TailInputTest < Test::Unit::TestCase
       mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
       plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
     end
+
+    data(
+      small: ["128", 128],
+      KiB: ["1k", 1024]
+    )
+    test 'max_line_size' do |(label, size)|
+      config = config_element("", "", {
+                                "tag" => "max_line_size",
+                                "path" => "#{TMP_DIR}/with_long_lines.txt",
+                                "format" => "none",
+                                "read_from_head" => true,
+                                "max_line_size" => label,
+                                "log_level" => "debug"
+                              })
+      File.open("#{TMP_DIR}/with_long_lines.txt", "w+") do |f|
+        f.puts "foo"
+        f.puts "x" * size # 'x' * size + \n > @max_line_size
+        f.puts "bar"
+      end
+      d = create_driver(config, false)
+      timestamp = Time.parse("Mon Nov 29 11:22:33 UTC 2021")
+      Timecop.freeze(timestamp)
+      d.run(expect_records: 2)
+      assert_equal([
+                     [{"message" => "foo"},{"message" => "bar"}],
+                     [
+                       "2021-11-29 11:22:33 +0000 [warn]: received line length is longer than #{size}\n",
+                       "2021-11-29 11:22:33 +0000 [debug]: skipped line: #{'x' * size}\n"
+                     ]
+                   ],
+                   [
+                     d.events.collect { |event| event.last },
+                     d.logs[-2..]
+                   ])
+    end
   end
 
   # Ensure that no fatal exception is raised when a file is missing and that

data/test/plugin/test_input.rb

@@ -73,7 +73,7 @@ class InputTest < Test::Unit::TestCase
 
     @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
 
-    assert{ @p.log.object_id != original_logger.object_id }
+    assert(@p.log.object_id != original_logger.object_id)
     assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
   end
 

data/test/plugin/test_out_exec.rb

@@ -243,7 +243,7 @@ class ExecOutputTest < Test::Unit::TestCase
   sub_test_case 'when executed process dies unexpectedly' do
     setup do
       @gen_config = ->(num){ <<EOC
-    command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write f.read} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
+    command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write(f.read); STDOUT.flush} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
     <inject>
       tag_key tag
       time_key time
@@ -265,7 +265,7 @@ EOC
       expect_path = "#{TMP_DIR}/fail_out"
 
       d.end_if{ File.exist?(expect_path) }
-      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
+      d.run(default_tag: 'test', flush: true, wait_flush_completion: true, shutdown: false) do
         d.feed(time, records[0])
         d.feed(time, records[1])
       end
@@ -281,7 +281,8 @@ EOC
       assert{ d.instance.buffer.queue.empty? }
       assert{ d.instance.dequeued_chunks.empty? }
 
-      d.instance_shutdown
+    ensure
+      d.instance_shutdown if d && d.instance
     end
 
     test 'flushed chunk will be taken back after child process unexpectedly exits' do
@@ -304,7 +305,8 @@ EOC
 
       assert{ File.exist?(expect_path) && File.size(expect_path) == 0 }
 
-      d.instance_shutdown
+    ensure
+      d.instance_shutdown if d && d.instance
     end
   end
 end

data/test/plugin/test_out_file.rb

@@ -394,6 +394,11 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal expect, result
   end
 
+  def check_result(path, expect)
+    result = File.read(path, mode: "rb")
+    assert_equal expect, result
+  end
+
   sub_test_case 'write' do
     test 'basic case' do
       d = create_driver
@@ -535,20 +540,27 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal 3, Dir.glob("#{TMP_DIR}/out_file_test.*").size
   end
 
-  test 'append' do
+  data(
+    "with compression" => true,
+    "without compression" => false,
+  )
+  test 'append' do |compression|
     time = event_time("2011-01-02 13:14:15 UTC")
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
 
     write_once = ->(){
-      d = create_driver %[
+      config = %[
         path #{TMP_DIR}/out_file_test
-        compress gz
         utc
         append true
         <buffer>
           timekey_use_utc true
         </buffer>
       ]
+      if compression
+        config << "        compress gz"
+      end
+      d = create_driver(config)
       d.run(default_tag: 'test'){
         d.feed(time, {"a"=>1})
         d.feed(time, {"a"=>2})
@@ -556,17 +568,21 @@ class FileOutputTest < Test::Unit::TestCase
       d.instance.last_written_path
     }
 
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 2)
+    log_file_name = "out_file_test.20110102.log"
+    if compression
+      log_file_name << ".gz"
+    end
 
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 3)
+    1.upto(3) do |i|
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/#{log_file_name}", path
+      expect = formatted_lines * i
+      if compression
+        check_gzipped_result(path, expect)
+      else
+        check_result(path, expect)
+      end
+    end
   end
 
   test 'append when JST' do

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -775,7 +775,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
       assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
 
-      assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
+      assert(@i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") })
     end
 
     test 'exponential backoff interval will be initialized when switched to secondary' do

data/test/plugin_helper/test_timer.rb

@@ -90,8 +90,8 @@ class TimerTest < Test::Unit::TestCase
     assert{ counter1 >= 4 && counter1 <= 5 }
     assert{ counter2 == 2 }
     msg = "Unexpected error raised. Stopping the timer. title=:t2"
-    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } }
-    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") } }
+    assert(d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") })
+    assert(d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") })
 
     d1.shutdown; d1.close; d1.terminate
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.14.3
+  version: 1.14.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-26 00:00:00.000000000 Z
+date: 2022-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -945,7 +945,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector