fluentd 1.16.5 → 1.16.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6557ce41f087cb6d7e0b7e8fa76d10259b8aa3538b917cb0b51bd4836cbdc01b
-  data.tar.gz: 062f08fb0fd8c9b6a8d0e5ff0f2619a94bc7b07de4d94d466c0a20c03459d8f3
+  metadata.gz: 538ca0cf96502680694da0004657405f47015b48d7202a80bfb5cd314fe1cb54
+  data.tar.gz: 5bd2f8742d676f6087de9d82e4c0cbaedb1eba1ee426e75f61e4c27422023874
 SHA512:
-  metadata.gz: 42c8786c9abbcb0144fc755492b41b2d6b1eb1dc77b5ffa3bce568554be262733659be42972b6b676dda52581924f29fc5976087368f865a629085a06264be83
-  data.tar.gz: ab0b41ca7db17f7968c512433efe185fb3228856fb53730af1888ed7ee6459962298607c2a87f1d3b722c51fe637a0638f119cf0722fb0a3087c0cf45e91ced8
+  metadata.gz: 7c836e75660911510111fb3b38002dfa7512332200ee9cbd2d3e22741d0cb1bb7476e2aeb1e0561968e166b56eb9f4e6c3854f1d831293e7d8ca29d7363191b6
+  data.tar.gz: 9c381f4b3b2b40eb332424f25b868521ccc940da822ba33998b950235d55668795cce0168e74eef91e65786b319a676ccc80211560348ecb2b6cbf5cc109cc63

data/.github/workflows/test.yml

@@ -3,8 +3,14 @@ name: Test
 on:
   push:
     branches: [v1.16]
+    paths-ignore:
+      - '*.md'
+      - 'lib/fluent/version.rb'
   pull_request:
     branches: [v1.16]
+    paths-ignore:
+      - '*.md'
+      - 'lib/fluent/version.rb'
 
 jobs:
   test:
@@ -29,4 +35,4 @@ jobs:
       - name: Install dependencies
         run: bundle install
       - name: Run tests
-        run: bundle exec rake test TESTOPTS=-v
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # v1.16
 
+## Release v1.16.6 - 2024/08/16
+
+### Bug Fix
+
+* YAML config syntax: Fix issue where `$log_level` element was not supported correctly
+  https://github.com/fluent/fluentd/pull/4486
+* parser_json: Fix wrong LoadError warning
+  https://github.com/fluent/fluentd/pull/4592
+* `fluentd` command: Fix `--plugin` (`-p`) option not to overwrite default value
+  https://github.com/fluent/fluentd/pull/4605
+
+### Misc
+
+* out_file: Add warn message for symlink_path setting
+  https://github.com/fluent/fluentd/pull/4512
+* Keep console gem v1.23 to avoid LoadError
+  https://github.com/fluent/fluentd/pull/4510
+
 ## Release v1.16.5 - 2024/03/27
 
 ### Bug Fix

data/fluentd.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", ["~> 1.4"])
+  gem.add_runtime_dependency("console", ["< 1.24"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -45,7 +46,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
   gem.add_development_dependency("rr", ["~> 3.0"])
-  gem.add_development_dependency("timecop", ["~> 0.9"])
+  # timecop v0.9.9 supports `Process.clock_gettime`. It breaks some tests.
+  # (https://github.com/fluent/fluentd/pull/4521)
+  gem.add_development_dependency("timecop", ["< 0.9.9"])
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])

data/lib/fluent/command/fluentd.rb

@@ -46,7 +46,7 @@ op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and
 }
 
 op.on('-p', '--plugin DIR', "add plugin directory") {|s|
-  (cmd_opts[:plugin_dirs] ||= []) << s
+  (cmd_opts[:plugin_dirs] ||= default_opts[:plugin_dirs]) << s
 }
 
 op.on('-I PATH', "add library path") {|s|

data/lib/fluent/config/yaml_parser/parser.rb

@@ -138,6 +138,10 @@ module Fluent
             sb.add_line('@id', v)
           end
 
+          if (v = config.delete('$log_level'))
+            sb.add_line('@log_level', v)
+          end
+
           config.each do |key, val|
             if val.is_a?(Array)
               val.each do |v|

data/lib/fluent/plugin/out_file.rb

@@ -172,6 +172,14 @@ module Fluent::Plugin
           log.warn "symlink_path is unavailable on Windows platform. disabled."
           @symlink_path = nil
         else
+          placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
+            begin
+              v.validate!
+            rescue Fluent::ConfigError => e
+              log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
+            end
+          end
+
           @buffer.extend SymlinkBufferMixin
           @buffer.symlink_path = @symlink_path
           @buffer.output_plugin_for_symlink = self

data/lib/fluent/plugin/parser_json.rb

@@ -50,23 +50,15 @@ module Fluent
       def configure_json_parser(name)
         case name
         when :oj
-          raise LoadError unless Fluent::OjOptions.available?
-          [Oj.method(:load), Oj::ParseError]
+          return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
+
+          log&.info "Oj is not installed, and failing back to Yajl for json parser"
+          configure_json_parser(:yajl)
         when :json then [JSON.method(:load), JSON::ParserError]
         when :yajl then [Yajl.method(:load), Yajl::ParseError]
         else
           raise "BUG: unknown json parser specified: #{name}"
         end
-      rescue LoadError => ex
-        name = :yajl
-        if log
-          if /\boj\z/.match?(ex.message)
-            log.info "Oj is not installed, and failing back to Yajl for json parser"
-          else
-            log.warn ex.message
-          end
-        end
-        retry
       end
 
       def parse(text)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.5'
+  VERSION = '1.16.6'
 
 end

data/test/command/test_fluentd.rb

@@ -128,11 +128,14 @@ class TestFluentdCommand < ::Test::Unit::TestCase
 
   # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
   # so `patterns_not_match` can test only logs up to that point.
+  # You can pass a block to assert something after log matching.
   def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
     matched = false
     matched_wrongly = false
-    assert_error_msg = ""
+    error_msg_match = ""
     stdio_buf = ""
+    succeeded_block = true
+    error_msg_block = ""
     begin
       execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
         begin
@@ -163,6 +166,13 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               end
             end
           end
+
+          begin
+            yield if block_given?
+          rescue => e
+            succeeded_block = false
+            error_msg_block = "failed block execution after matching: #{e}"
+          end
         ensure
           if SUPERVISOR_PID_PATTERN =~ stdio_buf
             @supervisor_pid = $1.to_i
@@ -173,19 +183,19 @@ class TestFluentdCommand < ::Test::Unit::TestCase
         end
       end
     rescue Timeout::Error
-      assert_error_msg = "execution timeout"
+      error_msg_match = "execution timeout"
       # https://github.com/fluent/fluentd/issues/4095
       # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
       # since the supervisor process will survive without being killed correctly.
       flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
     rescue => e
-      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
+      error_msg_match = "unexpected error in launching fluentd: #{e.inspect}"
     else
-      assert_error_msg = "log doesn't match" unless matched
+      error_msg_match = "log doesn't match" unless matched
     end
 
     if patterns_not_match.empty?
-      assert_error_msg = build_message(assert_error_msg,
+      error_msg_match = build_message(error_msg_match,
                                        "<?>\nwas expected to include:\n<?>",
                                        stdio_buf, pattern_list)
     else
@@ -197,16 +207,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
                             lines.any?{|line| line.include?(ptn) }
                           end
         if matched_wrongly
-          assert_error_msg << "\n" unless assert_error_msg.empty?
-          assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
+          error_msg_match << "\n" unless error_msg_match.empty?
+          error_msg_match << "pattern exists in logs wrongly: #{ptn}"
         end
       end
-      assert_error_msg = build_message(assert_error_msg,
+      error_msg_match = build_message(error_msg_match,
                                        "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
                                        stdio_buf, pattern_list, patterns_not_match)
     end
 
-    assert matched && !matched_wrongly, assert_error_msg
+    assert matched && !matched_wrongly, error_msg_match
+    assert succeeded_block, error_msg_block if block_given?
   end
 
   def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
@@ -1288,4 +1299,40 @@ CONF
                          "[debug]")
     end
   end
+
+  sub_test_case "plugin option" do
+    test "should be the default value when not specifying" do
+      conf_path = create_conf_file('test.conf', <<~CONF)
+        <source>
+          @type monitor_agent
+        </source>
+      CONF
+      assert File.exist?(conf_path)
+      cmdline = create_cmdline(conf_path)
+
+      assert_log_matches(cmdline, "fluentd worker is now running") do
+        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
+        actual_conf = JSON.parse(response)
+        assert_equal Fluent::Supervisor.default_options[:plugin_dirs], actual_conf["plugin_dirs"]
+      end
+    end
+
+    data(short: "-p")
+    data(long: "--plugin")
+    test "can be added by specifying the option" do |option_name|
+      conf_path = create_conf_file('test.conf', <<~CONF)
+        <source>
+          @type monitor_agent
+        </source>
+      CONF
+      assert File.exist?(conf_path)
+      cmdline = create_cmdline(conf_path, option_name, @tmp_dir, option_name, @tmp_dir)
+
+      assert_log_matches(cmdline, "fluentd worker is now running") do
+        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
+        actual_conf = JSON.parse(response)
+        assert_equal Fluent::Supervisor.default_options[:plugin_dirs] + [@tmp_dir, @tmp_dir], actual_conf["plugin_dirs"]
+      end
+    end
+  end
 end

data/test/plugin/in_tail/test_io_handler.rb

@@ -5,20 +5,19 @@ require 'fluent/plugin/metrics_local'
 require 'tempfile'
 
 class IntailIOHandlerTest < Test::Unit::TestCase
-  setup do
-    @file = Tempfile.new('intail_io_handler').binmode
-    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
-    opened_file_metrics.configure(config_element('metrics', '', {}))
-    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
-    closed_file_metrics.configure(config_element('metrics', '', {}))
-    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
-    rotated_file_metrics.configure(config_element('metrics', '', {}))
-    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
-  end
-
-  teardown do
-    @file.close rescue nil
-    @file.unlink rescue nil
+  def setup
+    Tempfile.create('intail_io_handler') do |file|
+      file.binmode
+      @file = file
+      opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+      opened_file_metrics.configure(config_element('metrics', '', {}))
+      closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+      closed_file_metrics.configure(config_element('metrics', '', {}))
+      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+      rotated_file_metrics.configure(config_element('metrics', '', {}))
+      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+      yield
+    end
   end
 
   def create_target_info

data/test/plugin/in_tail/test_position_file.rb

@@ -6,13 +6,12 @@ require 'fileutils'
 require 'tempfile'
 
 class IntailPositionFileTest < Test::Unit::TestCase
-  setup do
-    @file = Tempfile.new('intail_position_file_test').binmode
-  end
-
-  teardown do
-    @file.close rescue nil
-    @file.unlink rescue nil
+  def setup
+    Tempfile.create('intail_position_file_test') do |file|
+      file.binmode
+      @file = file
+      yield
+    end
   end
 
   UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION

data/test/plugin/test_out_file.rb

@@ -130,7 +130,7 @@ class FileOutputTest < Test::Unit::TestCase
           'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
           'add_path_suffix' => 'false',
           'append' => "true",
-          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
+          'symlink_path' => "#{TMP_DIR}/${tag}/conf_test.current.log",
           'compress' => 'gzip',
           'recompress' => 'true',
         }, [
@@ -183,6 +183,26 @@ class FileOutputTest < Test::Unit::TestCase
         Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
       end
     end
+
+    test 'warning for symlink_path not including correct placeholders corresponding to chunk keys' do
+      omit "Windows doesn't support symlink" if Fluent.windows?
+      conf = config_element('match', '**', {
+          'path' => "#{TMP_DIR}/${tag}/${key1}/${key2}/conf_test.%Y%m%d.%H%M.log",
+          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
+        }, [
+          config_element('buffer', 'time,tag,key1,key2', {
+              '@type' => 'file',
+              'timekey' => '1d',
+              'path' => "#{TMP_DIR}/buf_conf_test",
+          }),
+      ])
+      assert_nothing_raised do
+        d = create_driver(conf)
+        assert do
+          d.logs.count { |log| log.include?("multiple chunks are competing for a single symlink_path") } == 2
+        end
+      end
+    end
   end
 
   sub_test_case 'fully configured output' do

data/test/plugin/test_parser_json.rb

@@ -8,6 +8,37 @@ class JsonParserTest < ::Test::Unit::TestCase
     @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
   end
 
+  sub_test_case "configure_json_parser" do
+    data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
+    data("json", [:json, [JSON.method(:load), JSON::ParserError]])
+    data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
+    def test_return_each_loader((input, expected_return))
+      result = @parser.instance.configure_json_parser(input)
+      assert_equal expected_return, result
+    end
+
+    def test_raise_exception_for_unknown_input
+      assert_raise RuntimeError do
+        @parser.instance.configure_json_parser(:unknown)
+      end
+    end
+
+    def test_fall_back_oj_to_yajl_if_oj_not_available
+      stub(Fluent::OjOptions).available? { false }
+
+      result = @parser.instance.configure_json_parser(:oj)
+
+      assert_equal [Yajl.method(:load), Yajl::ParseError], result
+      logs = @parser.logs.collect do |log|
+        log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
+      end
+      assert_equal(
+        ["[info]: Oj is not installed, and failing back to Yajl for json parser\n"],
+        logs
+      )
+    end
+  end
+
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_parse(data)
     @parser.configure('json_parser' => data)

data/test/test_config.rb

@@ -167,6 +167,7 @@ class ConfigTest < Test::Unit::TestCase
             tag: tag.dummy
         - source:
             $type: tcp
+            $log_level: info
             tag: tag.tcp
             parse:
               $arg:
@@ -176,6 +177,7 @@ class ConfigTest < Test::Unit::TestCase
         - match:
             $tag: tag.*
             $type: stdout
+            $log_level: debug
             buffer:
               $type: memory
               flush_interval: 1s
@@ -208,10 +210,12 @@ class ConfigTest < Test::Unit::TestCase
           'tag.dummy',
           'tcp',
           'tag.tcp',
+          'info',
           'none',
           'why.parse.section.doesnot.have.arg,huh',
           'stdout',
           'tag.*',
+          'debug',
           'null',
           '**',
           '@FLUENT_LOG',
@@ -224,10 +228,12 @@ class ConfigTest < Test::Unit::TestCase
           dummy_source_conf['tag'],
           tcp_source_conf['@type'],
           tcp_source_conf['tag'],
+          tcp_source_conf['@log_level'],
           parse_tcp_conf['@type'],
           parse_tcp_conf.arg,
           match_conf['@type'],
           match_conf.arg,
+          match_conf['@log_level'],
           fluent_log_conf['@type'],
           fluent_log_conf.arg,
           label_conf.arg,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.16.5
+  version: 1.16.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-27 00:00:00.000000000 Z
+date: 2024-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -200,6 +200,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.24'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.24'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -274,16 +288,16 @@ dependencies:
   name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.9
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.9
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -941,7 +955,7 @@ homepage: https://www.fluentd.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -957,7 +971,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.4.19
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Fluentd event collector
 test_files: