fluentd 1.16.4-x86-mingw32 → 1.16.6-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55d9dff2b2c0dc6754261babb59c10b05cb4990b04366a10b6c3ed531e939ef6
-  data.tar.gz: 3fe6adb800c7f359b4dae8ede6c38710549da0bccd6181cc27c95cfa4ec5f776
+  metadata.gz: bf2c26992ab4316c7fdc3b1244a3cdcb80f071f0761d036946672478f480370d
+  data.tar.gz: 27690f3555b7e3c936b3f31a8e5994e52430360dd0a3239c04028e59fff684aa
 SHA512:
-  metadata.gz: eed2be6c185fc49df73504535018a7eaf7c5142a5fcebbc08d6e56cba93edc94d1c9ed159456bc52a6925c7e929c8bef4a2f2ca03850798ff52d90054eae849b
-  data.tar.gz: 5c2d5c814e6eeffdbafd831f0fa04ab79dc0f4979c3e3fa35dae941d7cec35c800b735f6c2efafacf91ae265e65d7d1d171ca941d28c60309399f74e61aad554
+  metadata.gz: ebc7ad56450ef9e1ecad23361c2863a56b2024afaea528a3b709d25b26077dde3c30b10dee44bbdf4c74e7d2ef6f58dd2136a3655e2904d55f16a01c206be8e9
+  data.tar.gz: c7a760d3db858034fb4d959538e852fab07221e732f4a49c4976b4d8382d2713ea8beba6628713291becc3060cde34f178fa54635fafa1745e347f35d307a877

data/.github/workflows/test.yml

@@ -3,8 +3,14 @@ name: Test
 on:
   push:
     branches: [v1.16]
+    paths-ignore:
+      - '*.md'
+      - 'lib/fluent/version.rb'
   pull_request:
     branches: [v1.16]
+    paths-ignore:
+      - '*.md'
+      - 'lib/fluent/version.rb'
 
 jobs:
   test:
@@ -29,4 +35,4 @@ jobs:
       - name: Install dependencies
         run: bundle install
       - name: Run tests
-        run: bundle exec rake test TESTOPTS=-v
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # v1.16
 
+## Release v1.16.6 - 2024/08/16
+
+### Bug Fix
+
+* YAML config syntax: Fix issue where `$log_level` element was not supported correctly
+  https://github.com/fluent/fluentd/pull/4486
+* parser_json: Fix wrong LoadError warning
+  https://github.com/fluent/fluentd/pull/4592
+* `fluentd` command: Fix `--plugin` (`-p`) option not to overwrite default value
+  https://github.com/fluent/fluentd/pull/4605
+
+### Misc
+
+* out_file: Add warn message for symlink_path setting
+  https://github.com/fluent/fluentd/pull/4512
+* Keep console gem v1.23 to avoid LoadError
+  https://github.com/fluent/fluentd/pull/4510
+
+## Release v1.16.5 - 2024/03/27
+
+### Bug Fix
+
+* Buffer: Fix emit error of v1.16.4 sometimes failing to process large data
+  exceeding chunk size limit
+  https://github.com/fluent/fluentd/pull/4447
+
 ## Release v1.16.4 - 2024/03/14
 
 ### Bug Fix

data/fluentd.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", ["~> 1.4"])
+  gem.add_runtime_dependency("console", ["< 1.24"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -45,7 +46,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
   gem.add_development_dependency("rr", ["~> 3.0"])
-  gem.add_development_dependency("timecop", ["~> 0.9"])
+  # timecop v0.9.9 supports `Process.clock_gettime`. It breaks some tests.
+  # (https://github.com/fluent/fluentd/pull/4521)
+  gem.add_development_dependency("timecop", ["< 0.9.9"])
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])

data/lib/fluent/command/fluentd.rb

@@ -46,7 +46,7 @@ op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and
 }
 
 op.on('-p', '--plugin DIR', "add plugin directory") {|s|
-  (cmd_opts[:plugin_dirs] ||= []) << s
+  (cmd_opts[:plugin_dirs] ||= default_opts[:plugin_dirs]) << s
 }
 
 op.on('-I PATH', "add library path") {|s|

data/lib/fluent/config/yaml_parser/parser.rb

@@ -138,6 +138,10 @@ module Fluent
             sb.add_line('@id', v)
           end
 
+          if (v = config.delete('$log_level'))
+            sb.add_line('@log_level', v)
+          end
+
           config.each do |key, val|
             if val.is_a?(Array)
               val.each do |v|

data/lib/fluent/plugin/buffer.rb

@@ -764,94 +764,95 @@ module Fluent
         while writing_splits_index < splits.size
           chunk = get_next_chunk.call
           errors = []
+          # The chunk must be locked until being passed to &block.
+          chunk.mon_enter
           modified_chunks << {chunk: chunk, adding_bytesize: 0, errors: errors}
-          chunk.synchronize do
-            raise ShouldRetry unless chunk.writable?
-            staged_chunk_used = true if chunk.staged?
-
-            original_bytesize = committed_bytesize = chunk.bytesize
-            begin
-              while writing_splits_index < splits.size
-                split = splits[writing_splits_index]
-                formatted_split = format ? format.call(split) : nil
 
-                if split.size == 1 # Check BufferChunkOverflowError
-                  determined_bytesize = nil
-                  if @compress != :text
-                    determined_bytesize = nil
-                  elsif formatted_split
-                    determined_bytesize = formatted_split.bytesize
-                  elsif split.first.respond_to?(:bytesize)
-                    determined_bytesize = split.first.bytesize
-                  end
+          raise ShouldRetry unless chunk.writable?
+          staged_chunk_used = true if chunk.staged?
 
-                  if determined_bytesize && determined_bytesize > @chunk_limit_size
-                    # It is a obvious case that BufferChunkOverflowError should be raised here.
-                    # But if it raises here, already processed 'split' or
-                    # the proceeding 'split' will be lost completely.
-                    # So it is a last resort to delay raising such a exception
-                    errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                    writing_splits_index += 1
-                    next
-                  end
+          original_bytesize = committed_bytesize = chunk.bytesize
+          begin
+            while writing_splits_index < splits.size
+              split = splits[writing_splits_index]
+              formatted_split = format ? format.call(split) : nil
 
-                  if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
-                    # The split will (might) cause size over so keep already processed
-                    # 'split' content here (allow performance regression a bit).
-                    chunk.commit
-                    committed_bytesize = chunk.bytesize
-                  end
+              if split.size == 1 # Check BufferChunkOverflowError
+                determined_bytesize = nil
+                if @compress != :text
+                  determined_bytesize = nil
+                elsif formatted_split
+                  determined_bytesize = formatted_split.bytesize
+                elsif split.first.respond_to?(:bytesize)
+                  determined_bytesize = split.first.bytesize
                 end
 
-                if format
-                  chunk.concat(formatted_split, split.size)
-                else
-                  chunk.append(split, compress: @compress)
+                if determined_bytesize && determined_bytesize > @chunk_limit_size
+                  # It is a obvious case that BufferChunkOverflowError should be raised here.
+                  # But if it raises here, already processed 'split' or
+                  # the proceeding 'split' will be lost completely.
+                  # So it is a last resort to delay raising such a exception
+                  errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                  writing_splits_index += 1
+                  next
                 end
-                adding_bytes = chunk.bytesize - committed_bytesize
 
-                if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
-                  chunk.rollback
+                if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
+                  # The split will (might) cause size over so keep already processed
+                  # 'split' content here (allow performance regression a bit).
+                  chunk.commit
                   committed_bytesize = chunk.bytesize
+                end
+              end
 
-                  if split.size == 1 # Check BufferChunkOverflowError again
-                    if adding_bytes > @chunk_limit_size
-                      errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                      writing_splits_index += 1
-                      next
-                    else
-                      # As already processed content is kept after rollback, then unstaged chunk should be queued.
-                      # After that, re-process current split again.
-                      # New chunk should be allocated, to do it, modify @stage and so on.
-                      synchronize { @stage.delete(modified_metadata) }
-                      staged_chunk_used = false
-                      chunk.unstaged!
-                      break
-                    end
-                  end
+              if format
+                chunk.concat(formatted_split, split.size)
+              else
+                chunk.append(split, compress: @compress)
+              end
+              adding_bytes = chunk.bytesize - committed_bytesize
 
-                  if chunk_size_full?(chunk) || split.size == 1
-                    enqueue_chunk_before_retry = true
+              if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
+                chunk.rollback
+                committed_bytesize = chunk.bytesize
+
+                if split.size == 1 # Check BufferChunkOverflowError again
+                  if adding_bytes > @chunk_limit_size
+                    errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                    writing_splits_index += 1
+                    next
                   else
-                    splits_count *= 10
+                    # As already processed content is kept after rollback, then unstaged chunk should be queued.
+                    # After that, re-process current split again.
+                    # New chunk should be allocated, to do it, modify @stage and so on.
+                    synchronize { @stage.delete(modified_metadata) }
+                    staged_chunk_used = false
+                    chunk.unstaged!
+                    break
                   end
+                end
 
-                  raise ShouldRetry
+                if chunk_size_full?(chunk) || split.size == 1
+                  enqueue_chunk_before_retry = true
+                else
+                  splits_count *= 10
                 end
 
-                writing_splits_index += 1
+                raise ShouldRetry
+              end
 
-                if chunk_size_full?(chunk)
-                  break
-                end
+              writing_splits_index += 1
+
+              if chunk_size_full?(chunk)
+                break
               end
-            rescue
-              chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
-              raise
             end
-
-            modified_chunks.last[:adding_bytesize] = chunk.bytesize - original_bytesize
+          rescue
+            chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
+            raise
           end
+
+          modified_chunks.last[:adding_bytesize] = chunk.bytesize - original_bytesize
         end
         modified_chunks.each do |data|
           block.call(data[:chunk], data[:adding_bytesize], data[:errors])
@@ -863,9 +864,15 @@ module Fluent
           if chunk.unstaged?
             chunk.purge rescue nil
           end
+          chunk.mon_exit rescue nil
         end
         enqueue_chunk(metadata) if enqueue_chunk_before_retry
         retry
+      ensure
+        modified_chunks.each do |data|
+          chunk = data[:chunk]
+          chunk.mon_exit
+        end
       end
 
       STATS_KEYS = [

data/lib/fluent/plugin/out_file.rb

@@ -172,6 +172,14 @@ module Fluent::Plugin
           log.warn "symlink_path is unavailable on Windows platform. disabled."
           @symlink_path = nil
         else
+          placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
+            begin
+              v.validate!
+            rescue Fluent::ConfigError => e
+              log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
+            end
+          end
+
           @buffer.extend SymlinkBufferMixin
           @buffer.symlink_path = @symlink_path
           @buffer.output_plugin_for_symlink = self

data/lib/fluent/plugin/parser_json.rb

@@ -50,23 +50,15 @@ module Fluent
       def configure_json_parser(name)
         case name
         when :oj
-          raise LoadError unless Fluent::OjOptions.available?
-          [Oj.method(:load), Oj::ParseError]
+          return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
+
+          log&.info "Oj is not installed, and failing back to Yajl for json parser"
+          configure_json_parser(:yajl)
         when :json then [JSON.method(:load), JSON::ParserError]
         when :yajl then [Yajl.method(:load), Yajl::ParseError]
         else
           raise "BUG: unknown json parser specified: #{name}"
         end
-      rescue LoadError => ex
-        name = :yajl
-        if log
-          if /\boj\z/.match?(ex.message)
-            log.info "Oj is not installed, and failing back to Yajl for json parser"
-          else
-            log.warn ex.message
-          end
-        end
-        retry
       end
 
       def parse(text)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.4'
+  VERSION = '1.16.6'
 
 end

data/test/command/test_fluentd.rb

@@ -128,11 +128,14 @@ class TestFluentdCommand < ::Test::Unit::TestCase
 
   # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
   # so `patterns_not_match` can test only logs up to that point.
+  # You can pass a block to assert something after log matching.
   def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
     matched = false
     matched_wrongly = false
-    assert_error_msg = ""
+    error_msg_match = ""
     stdio_buf = ""
+    succeeded_block = true
+    error_msg_block = ""
     begin
       execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
         begin
@@ -163,6 +166,13 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               end
             end
           end
+
+          begin
+            yield if block_given?
+          rescue => e
+            succeeded_block = false
+            error_msg_block = "failed block execution after matching: #{e}"
+          end
         ensure
           if SUPERVISOR_PID_PATTERN =~ stdio_buf
             @supervisor_pid = $1.to_i
@@ -173,19 +183,19 @@ class TestFluentdCommand < ::Test::Unit::TestCase
         end
       end
     rescue Timeout::Error
-      assert_error_msg = "execution timeout"
+      error_msg_match = "execution timeout"
       # https://github.com/fluent/fluentd/issues/4095
       # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
       # since the supervisor process will survive without being killed correctly.
       flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
     rescue => e
-      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
+      error_msg_match = "unexpected error in launching fluentd: #{e.inspect}"
     else
-      assert_error_msg = "log doesn't match" unless matched
+      error_msg_match = "log doesn't match" unless matched
     end
 
     if patterns_not_match.empty?
-      assert_error_msg = build_message(assert_error_msg,
+      error_msg_match = build_message(error_msg_match,
                                        "<?>\nwas expected to include:\n<?>",
                                        stdio_buf, pattern_list)
     else
@@ -197,16 +207,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
                             lines.any?{|line| line.include?(ptn) }
                           end
         if matched_wrongly
-          assert_error_msg << "\n" unless assert_error_msg.empty?
-          assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
+          error_msg_match << "\n" unless error_msg_match.empty?
+          error_msg_match << "pattern exists in logs wrongly: #{ptn}"
         end
       end
-      assert_error_msg = build_message(assert_error_msg,
+      error_msg_match = build_message(error_msg_match,
                                        "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
                                        stdio_buf, pattern_list, patterns_not_match)
     end
 
-    assert matched && !matched_wrongly, assert_error_msg
+    assert matched && !matched_wrongly, error_msg_match
+    assert succeeded_block, error_msg_block if block_given?
   end
 
   def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
@@ -1288,4 +1299,40 @@ CONF
                          "[debug]")
     end
   end
+
+  sub_test_case "plugin option" do
+    test "should be the default value when not specifying" do
+      conf_path = create_conf_file('test.conf', <<~CONF)
+        <source>
+          @type monitor_agent
+        </source>
+      CONF
+      assert File.exist?(conf_path)
+      cmdline = create_cmdline(conf_path)
+
+      assert_log_matches(cmdline, "fluentd worker is now running") do
+        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
+        actual_conf = JSON.parse(response)
+        assert_equal Fluent::Supervisor.default_options[:plugin_dirs], actual_conf["plugin_dirs"]
+      end
+    end
+
+    data(short: "-p")
+    data(long: "--plugin")
+    test "can be added by specifying the option" do |option_name|
+      conf_path = create_conf_file('test.conf', <<~CONF)
+        <source>
+          @type monitor_agent
+        </source>
+      CONF
+      assert File.exist?(conf_path)
+      cmdline = create_cmdline(conf_path, option_name, @tmp_dir, option_name, @tmp_dir)
+
+      assert_log_matches(cmdline, "fluentd worker is now running") do
+        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
+        actual_conf = JSON.parse(response)
+        assert_equal Fluent::Supervisor.default_options[:plugin_dirs] + [@tmp_dir, @tmp_dir], actual_conf["plugin_dirs"]
+      end
+    end
+  end
 end

data/test/plugin/in_tail/test_io_handler.rb

@@ -5,20 +5,19 @@ require 'fluent/plugin/metrics_local'
 require 'tempfile'
 
 class IntailIOHandlerTest < Test::Unit::TestCase
-  setup do
-    @file = Tempfile.new('intail_io_handler').binmode
-    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
-    opened_file_metrics.configure(config_element('metrics', '', {}))
-    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
-    closed_file_metrics.configure(config_element('metrics', '', {}))
-    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
-    rotated_file_metrics.configure(config_element('metrics', '', {}))
-    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
-  end
-
-  teardown do
-    @file.close rescue nil
-    @file.unlink rescue nil
+  def setup
+    Tempfile.create('intail_io_handler') do |file|
+      file.binmode
+      @file = file
+      opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+      opened_file_metrics.configure(config_element('metrics', '', {}))
+      closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+      closed_file_metrics.configure(config_element('metrics', '', {}))
+      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+      rotated_file_metrics.configure(config_element('metrics', '', {}))
+      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+      yield
+    end
   end
 
   def create_target_info

data/test/plugin/in_tail/test_position_file.rb

@@ -6,13 +6,12 @@ require 'fileutils'
 require 'tempfile'
 
 class IntailPositionFileTest < Test::Unit::TestCase
-  setup do
-    @file = Tempfile.new('intail_position_file_test').binmode
-  end
-
-  teardown do
-    @file.close rescue nil
-    @file.unlink rescue nil
+  def setup
+    Tempfile.create('intail_position_file_test') do |file|
+      file.binmode
+      @file = file
+      yield
+    end
   end
 
   UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION

data/test/plugin/test_buffer.rb

@@ -901,6 +901,65 @@ class BufferTest < Test::Unit::TestCase
 
       assert_equal 2, purge_count
     end
+
+    # https://github.com/fluent/fluentd/issues/4446
+    test "#write_step_by_step keeps chunks kept in locked in entire #write process" do
+      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
+      assert_equal 0.95, @p.chunk_full_threshold
+
+      mon_enter_counts_by_chunk = {}
+      mon_exit_counts_by_chunk = {}
+
+      stub.proxy(@p).generate_chunk(anything) do |chunk|
+        stub(chunk).mon_enter do
+          enter_count = 1 + mon_enter_counts_by_chunk.fetch(chunk, 0)
+          exit_count = mon_exit_counts_by_chunk.fetch(chunk, 0)
+          mon_enter_counts_by_chunk[chunk] = enter_count
+
+          # Assert that chunk is passed to &block of write_step_by_step before exiting the lock.
+          # (i.e. The lock count must be 2 greater than the exit count).
+          # Since ShouldRetry occurs once, the staged chunk takes the lock 3 times when calling the block.
+          if chunk.staged?
+            lock_in_block = enter_count == 3
+            assert_equal(enter_count - 2, exit_count) if lock_in_block
+          else
+            lock_in_block = enter_count == 2
+            assert_equal(enter_count - 2, exit_count) if lock_in_block
+          end
+        end
+        stub(chunk).mon_exit do
+          exit_count = 1 + mon_exit_counts_by_chunk.fetch(chunk, 0)
+          mon_exit_counts_by_chunk[chunk] = exit_count
+        end
+        chunk
+      end
+
+      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
+      small_row = "x" * 1024 * 400
+      big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it does't cause BufferOverFlowError.
+
+      # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`
+      @p.write({m => [small_row] * 40 + [big_row] + ["x"]})
+
+      # Above event strem will be splitted twice by `Buffer#write_step_by_step`
+      #
+      # 1. `write_once`: 42 [events] * 1 [stream]
+      # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
+      # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
+      # 
+      # Example of staged chunk lock behavior:
+      # 
+      # 1. mon_enter in write_step_by_step
+      # 2. ShouldRetry occurs
+      # 3. mon_exit in write_step_by_step
+      # 4. mon_enter again in write_step_by_step (retry)
+      # 5. passed to &block of write_step_by_step
+      # 6. mon_enter in the block (write)
+      # 7. mon_exit in write_step_by_step
+      # 8. mon_exit in write
+
+      assert_equal(mon_enter_counts_by_chunk.values, mon_exit_counts_by_chunk.values)
+    end
   end
 
   sub_test_case 'standard format with configuration for test with lower chunk limit size' do

data/test/plugin/test_out_file.rb

@@ -130,7 +130,7 @@ class FileOutputTest < Test::Unit::TestCase
           'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
           'add_path_suffix' => 'false',
           'append' => "true",
-          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
+          'symlink_path' => "#{TMP_DIR}/${tag}/conf_test.current.log",
           'compress' => 'gzip',
           'recompress' => 'true',
         }, [
@@ -183,6 +183,26 @@ class FileOutputTest < Test::Unit::TestCase
         Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
       end
     end
+
+    test 'warning for symlink_path not including correct placeholders corresponding to chunk keys' do
+      omit "Windows doesn't support symlink" if Fluent.windows?
+      conf = config_element('match', '**', {
+          'path' => "#{TMP_DIR}/${tag}/${key1}/${key2}/conf_test.%Y%m%d.%H%M.log",
+          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
+        }, [
+          config_element('buffer', 'time,tag,key1,key2', {
+              '@type' => 'file',
+              'timekey' => '1d',
+              'path' => "#{TMP_DIR}/buf_conf_test",
+          }),
+      ])
+      assert_nothing_raised do
+        d = create_driver(conf)
+        assert do
+          d.logs.count { |log| log.include?("multiple chunks are competing for a single symlink_path") } == 2
+        end
+      end
+    end
   end
 
   sub_test_case 'fully configured output' do

data/test/plugin/test_parser_json.rb

@@ -8,6 +8,37 @@ class JsonParserTest < ::Test::Unit::TestCase
     @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
   end
 
+  sub_test_case "configure_json_parser" do
+    data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
+    data("json", [:json, [JSON.method(:load), JSON::ParserError]])
+    data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
+    def test_return_each_loader((input, expected_return))
+      result = @parser.instance.configure_json_parser(input)
+      assert_equal expected_return, result
+    end
+
+    def test_raise_exception_for_unknown_input
+      assert_raise RuntimeError do
+        @parser.instance.configure_json_parser(:unknown)
+      end
+    end
+
+    def test_fall_back_oj_to_yajl_if_oj_not_available
+      stub(Fluent::OjOptions).available? { false }
+
+      result = @parser.instance.configure_json_parser(:oj)
+
+      assert_equal [Yajl.method(:load), Yajl::ParseError], result
+      logs = @parser.logs.collect do |log|
+        log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
+      end
+      assert_equal(
+        ["[info]: Oj is not installed, and failing back to Yajl for json parser\n"],
+        logs
+      )
+    end
+  end
+
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_parse(data)
     @parser.configure('json_parser' => data)

data/test/test_config.rb

@@ -167,6 +167,7 @@ class ConfigTest < Test::Unit::TestCase
             tag: tag.dummy
         - source:
             $type: tcp
+            $log_level: info
             tag: tag.tcp
             parse:
               $arg:
@@ -176,6 +177,7 @@ class ConfigTest < Test::Unit::TestCase
         - match:
             $tag: tag.*
             $type: stdout
+            $log_level: debug
             buffer:
               $type: memory
               flush_interval: 1s
@@ -208,10 +210,12 @@ class ConfigTest < Test::Unit::TestCase
           'tag.dummy',
           'tcp',
           'tag.tcp',
+          'info',
           'none',
           'why.parse.section.doesnot.have.arg,huh',
           'stdout',
           'tag.*',
+          'debug',
           'null',
           '**',
           '@FLUENT_LOG',
@@ -224,10 +228,12 @@ class ConfigTest < Test::Unit::TestCase
           dummy_source_conf['tag'],
           tcp_source_conf['@type'],
           tcp_source_conf['tag'],
+          tcp_source_conf['@log_level'],
           parse_tcp_conf['@type'],
           parse_tcp_conf.arg,
           match_conf['@type'],
           match_conf.arg,
+          match_conf['@log_level'],
           fluent_log_conf['@type'],
           fluent_log_conf.arg,
           label_conf.arg,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.16.4
+  version: 1.16.6
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-14 00:00:00.000000000 Z
+date: 2024-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -200,6 +200,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.24'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.24'
 - !ruby/object:Gem::Dependency
   name: win32-service
   requirement: !ruby/object:Gem::Requirement
@@ -330,16 +344,16 @@ dependencies:
   name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.9
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.9
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -997,7 +1011,7 @@ homepage: https://www.fluentd.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -1013,7 +1027,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.4.19
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Fluentd event collector
 test_files: