fluentd 1.16.6 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 538ca0cf96502680694da0004657405f47015b48d7202a80bfb5cd314fe1cb54
-  data.tar.gz: 5bd2f8742d676f6087de9d82e4c0cbaedb1eba1ee426e75f61e4c27422023874
+  metadata.gz: df7f40114152fe9cd139ef2c195e7baf19b501a3e949ba8412de445a3337c26f
+  data.tar.gz: e7b728c294b2797223405571ffc66853fb21210fe696b5299179c8a85f1c51a0
 SHA512:
-  metadata.gz: 7c836e75660911510111fb3b38002dfa7512332200ee9cbd2d3e22741d0cb1bb7476e2aeb1e0561968e166b56eb9f4e6c3854f1d831293e7d8ca29d7363191b6
-  data.tar.gz: 9c381f4b3b2b40eb332424f25b868521ccc940da822ba33998b950235d55668795cce0168e74eef91e65786b319a676ccc80211560348ecb2b6cbf5cc109cc63
+  metadata.gz: ad48b5478ae93b0c41e6db5cec80b1e67ac18c24b44ce43193143150d8fbf765b3b2674ddc6e32dfd649095647bb6113687c436fa463249f0e1ab64c665ab7b3
+  data.tar.gz: 4cfc5bb3ecd1b116346d6d899c243142c4c4a8768e980f895a3d5b6ee3a49b5b7fc132710316a657fb6353651b7fe42345371d536ed6123d52bce3a79bd4ff35

data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml

@@ -0,0 +1,50 @@
+title: "[QA (Japanese)]"
+labels: ["Q&A (Japanese)"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        日本語で気軽に質問するためのカテゴリです。もし他の人が困っているのを見つけたらぜひ回答してあげてください。
+  - type: textarea
+    id: question
+    attributes:
+      label: やりたいこと
+      description: |
+        何について困っているのかを書いてください。試したことや実際の結果を示してください。
+        期待する挙動と実際の結果の違いがあればそれも書くのをおすすめします。
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: 設定した内容
+      description: |
+        どのような設定をして期待する挙動を実現しようとしたのかを書いてください。(例: fluentd.confの内容を貼り付ける)
+      render: apache
+  - type: textarea
+    id: logs
+    attributes:
+      label: ログの内容
+      description: |
+        Fluentdのログを提示してください。エラーログがあると回答の助けになります。(例: fluentd.logの内容を貼り付ける)
+      render: shell
+  - type: textarea
+    id: environment
+    attributes:
+      label: 環境について
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        どんな環境で困っているかの情報がないと、再現できないため誰も回答できないことがあります。
+        必要な情報を記入することをおすすめします。
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Fluent Package version:
+        - Docker image (tag):
+        - Operating system:
+        - Kernel version:
+      render: markdown

data/.github/DISCUSSION_TEMPLATE/q-a.yml

@@ -0,0 +1,47 @@
+title: "[Q&A]"
+labels: ["Q&A"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        It is recommended to support each other.
+  - type: textarea
+    id: question
+    attributes:
+      label: What is a problem?
+      description: |
+        A clear and concise description of what you want to happen.
+        What exactly did you do (or not do) that was effective (or ineffective)?
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Describe the configuration of Fluentd
+      description: |
+        If there is the actual configuration of Fluentd, it will help.
+  - type: textarea
+    id: logs
+    attributes:
+      label: Describe the logs of Fluentd
+      description: |
+        If there are error logs of Fluentd, it will help.
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Please describe your environment information. If will help to support.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Fluent Package version:
+        - Docker image (tag):
+        - Operating system:
+        - Kernel version:
+      render: markdown

data/.github/workflows/test-ruby-head.yml

@@ -0,0 +1,31 @@
+name: Test with Ruby head
+
+on:
+  schedule:
+    - cron: '11 14 * * 0'
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        ruby-version: ['head']
+
+    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install addons
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: sudo apt-get install libgmp3-dev libcap-ng-dev
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/.github/workflows/test.yml

@@ -2,15 +2,9 @@ name: Test
 
 on:
   push:
-    branches: [v1.16]
-    paths-ignore:
-      - '*.md'
-      - 'lib/fluent/version.rb'
+    branches: [master]
   pull_request:
-    branches: [v1.16]
-    paths-ignore:
-      - '*.md'
-      - 'lib/fluent/version.rb'
+    branches: [master]
 
 jobs:
   test:

data/CHANGELOG.md

@@ -1,22 +1,46 @@
-# v1.16
+# v1.17
+
+## Release v1.17.0 - 2024/04/30
+
+### Enhancement
 
-## Release v1.16.6 - 2024/08/16
+* in_http: Recognize CSP reports as JSON data
+  https://github.com/fluent/fluentd/pull/4282
+* out_http: Add option to reuse connections
+  https://github.com/fluent/fluentd/pull/4330
+* in_tail: Expand glob capability for square brackets and one character matcher
+  https://github.com/fluent/fluentd/pull/4401
+* out_http: Support AWS Signature Version 4 authentication
+  https://github.com/fluent/fluentd/pull/4459
 
 ### Bug Fix
 
-* YAML config syntax: Fix issue where `$log_level` element was not supported correctly
-  https://github.com/fluent/fluentd/pull/4486
-* parser_json: Fix wrong LoadError warning
-  https://github.com/fluent/fluentd/pull/4592
-* `fluentd` command: Fix `--plugin` (`-p`) option not to overwrite default value
-  https://github.com/fluent/fluentd/pull/4605
+* Make sure `parser_json` and `parser_msgpack` return `Hash`.
+  Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`.
+  https://github.com/fluent/fluentd/pull/4474
+* filter_parser: Add error event for multiple parsed results
+  https://github.com/fluent/fluentd/pull/4478
 
 ### Misc
 
-* out_file: Add warn message for symlink_path setting
-  https://github.com/fluent/fluentd/pull/4512
-* Keep console gem v1.23 to avoid LoadError
-  https://github.com/fluent/fluentd/pull/4510
+* Raise minimum required ruby version
+  https://github.com/fluent/fluentd/pull/4288
+* Require missing dependent gems as of Ruby 3.4-dev
+  https://github.com/fluent/fluentd/pull/4411
+* Minor code refactoring
+  https://github.com/fluent/fluentd/pull/4294
+  https://github.com/fluent/fluentd/pull/4299
+  https://github.com/fluent/fluentd/pull/4302
+  https://github.com/fluent/fluentd/pull/4320
+* CI fixes
+  https://github.com/fluent/fluentd/pull/4369
+  https://github.com/fluent/fluentd/pull/4433
+  https://github.com/fluent/fluentd/pull/4452
+  https://github.com/fluent/fluentd/pull/4477
+* github: unify YAML file extension to .yml
+  https://github.com/fluent/fluentd/pull/4429
+
+# v1.16
 
 ## Release v1.16.5 - 2024/03/27
 

data/README.md

@@ -29,7 +29,7 @@ Fluentd: Open-Source Log Collector
 
 ### Prerequisites
 
-- Ruby 2.4 or later
+- Ruby 2.7 or later
 - git
 
 `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.

data/Rakefile

@@ -13,7 +13,7 @@ task test: [:base_test]
 namespace :build do
   desc 'Build gems for all platforms'
   task :all do
-    Bundler.with_clean_env do
+    Bundler.with_original_env do
       %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
         ENV['GEM_BUILD_FAKE_PLATFORM'] = name
         Rake::Task["build"].execute

data/fluentd.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 2.4'
+  gem.required_ruby_version = '>= 2.7'
 
   gem.add_runtime_dependency("bundler")
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
@@ -29,7 +29,11 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", ["~> 1.4"])
-  gem.add_runtime_dependency("console", ["< 1.24"])
+
+  # gems that aren't default gems as of Ruby 3.4
+  gem.add_runtime_dependency("base64", ["~> 0.2"])
+  gem.add_runtime_dependency("csv", ["~> 3.2"])
+  gem.add_runtime_dependency("drb", ["~> 2.2"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -46,12 +50,13 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
   gem.add_development_dependency("rr", ["~> 3.0"])
-  # timecop v0.9.9 supports `Process.clock_gettime`. It breaks some tests.
-  # (https://github.com/fluent/fluentd/pull/4521)
-  gem.add_development_dependency("timecop", ["< 0.9.9"])
+  gem.add_development_dependency("timecop", ["~> 0.9"])
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
   gem.add_development_dependency("async", "~> 1.23")
   gem.add_development_dependency("async-http", ">= 0.50.0")
+  gem.add_development_dependency("aws-sigv4", ["~> 1.8"])
+  gem.add_development_dependency("aws-sdk-core", ["~> 3.191"])
+  gem.add_development_dependency("rexml", ["~> 3.2"])
 end

data/lib/fluent/command/binlog_reader.rb

@@ -130,7 +130,7 @@ module BinlogReaderCommand
     private
 
     def configure_option_parser
-      @options.merge!(config_params: {})
+      @options[:config_params] = {}
 
       @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
 

data/lib/fluent/command/fluentd.rb

@@ -46,7 +46,7 @@ op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and
 }
 
 op.on('-p', '--plugin DIR', "add plugin directory") {|s|
-  (cmd_opts[:plugin_dirs] ||= default_opts[:plugin_dirs]) << s
+  (cmd_opts[:plugin_dirs] ||= []) << s
 }
 
 op.on('-I PATH', "add library path") {|s|

data/lib/fluent/config/configure_proxy.rb

@@ -201,7 +201,7 @@ module Fluent
 
       def overwrite_defaults(other) # other is owner plugin's corresponding proxy
         self.defaults = self.defaults.merge(other.defaults)
-        self.sections.keys.each do |section_key|
+        self.sections.each_key do |section_key|
           if other.sections.has_key?(section_key)
             self.sections[section_key].overwrite_defaults(other.sections[section_key])
           end
@@ -274,7 +274,7 @@ module Fluent
         option_value_type!(name, opts, :deprecated, String)
         option_value_type!(name, opts, :obsoleted, String)
         if type == :enum
-          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
+          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?(Symbol)
             raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
           end
         end

data/lib/fluent/config/types.rb

@@ -123,7 +123,7 @@ module Fluent
 
       s = val.to_sym
       list = opts[:list]
-      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
+      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?(Symbol)
       unless list.include?(s)
         raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
       end

data/lib/fluent/config/yaml_parser/parser.rb

@@ -138,10 +138,6 @@ module Fluent
             sb.add_line('@id', v)
           end
 
-          if (v = config.delete('$log_level'))
-            sb.add_line('@log_level', v)
-          end
-
           config.each do |key, val|
             if val.is_a?(Array)
               val.each do |v|

data/lib/fluent/configurable.rb

@@ -31,13 +31,13 @@ module Fluent
       super
       # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
       proxy = self.class.merged_configure_proxy
-      proxy.params.keys.each do |name|
+      proxy.params.each_key do |name|
         next if name.to_s.start_with?('@')
         if proxy.defaults.has_key?(name)
           instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
         end
       end
-      proxy.sections.keys.each do |name|
+      proxy.sections.each_key do |name|
         next if name.to_s.start_with?('@')
         subproxy = proxy.sections[name]
         if subproxy.multi?

data/lib/fluent/counter/mutex_hash.rb

@@ -54,7 +54,7 @@ module Fluent
               if mutex.try_lock
                 locks[key] = mutex
               else
-                locks.values.each(&:unlock)
+                locks.each_value(&:unlock)
                 locks = {}          # flush locked keys
                 break
               end

data/lib/fluent/fluent_log_event_router.rb

@@ -47,8 +47,6 @@ module Fluent
         # it's not suppressed in default event router for non-log-event events
         log_event_router.suppress_missing_match!
 
-        log_event_router = log_event_router
-
         unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
         unless unmatched_tags.empty?
           $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags

data/lib/fluent/plugin/buf_file.rb

@@ -56,7 +56,7 @@ module Fluent
 
         @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
 
-        multi_workers_configured = owner.system_config.workers > 1 ? true : false
+        multi_workers_configured = owner.system_config.workers > 1
 
         using_plugin_root_dir = false
         unless @path

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -399,7 +399,7 @@ module Fluent
           end
 
           if chunk.slice(0, 2) == BUFFER_HEADER
-            size = chunk.slice(2, 4).unpack('N').first
+            size = chunk.slice(2, 4).unpack1('N')
             if size
               return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
             end

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -238,17 +238,16 @@ module Fluent
           callback.call(file) if callback
         end
 
-        URI_PARSER = URI::Parser.new
         ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
 
         def encode_key(metadata)
           k = @key ? metadata.variables[@key] : metadata.tag
           k ||= ''
-          URI_PARSER.escape(k, ESCAPE_REGEXP)
+          URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
         end
 
         def decode_key(key)
-          URI_PARSER.unescape(key)
+          URI::DEFAULT_PARSER.unescape(key)
         end
 
         def create_new_chunk(path, metadata, perm)

data/lib/fluent/plugin/filter_parser.rb

@@ -70,6 +70,13 @@ module Fluent::Plugin
         end
       end
       begin
+        # Note: https://github.com/fluent/fluentd/issues/4100
+        # If the parser returns multiple records from one raw_value,
+        # this returns only the first one record.
+        # This should be fixed in the future version.
+        result_time = nil
+        result_record = nil
+
         @parser.parse(raw_value) do |t, values|
           if values
             t = if @reserve_time
@@ -79,20 +86,31 @@ module Fluent::Plugin
                 end
             @accessor.delete(record) if @remove_key_name_field
             r = handle_parsed(tag, record, t, values)
-            return t, r
+
+            if result_record.nil?
+              result_time = t
+              result_record = r
+            else
+              if @emit_invalid_record_to_error
+                router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
+                  "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
+                ))
+              end
+            end
           else
             if @emit_invalid_record_to_error
               router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
             end
-            if @reserve_data
-              t = time
-              r = handle_parsed(tag, record, time, {})
-              return t, r
-            else
-              return FAILED_RESULT
-            end
+
+            next unless @reserve_data
+            next unless result_record.nil?
+
+            result_time = time
+            result_record = handle_parsed(tag, record, time, {})
           end
         end
+
+        return result_time, result_record
       rescue Fluent::Plugin::Parser::ParserError => e
         if @emit_invalid_record_to_error
           raise e

data/lib/fluent/plugin/in_http.rb

@@ -203,54 +203,24 @@ module Fluent::Plugin
       begin
         path = path_info[1..-1]  # remove /
         tag = path.split('/').join('.')
-        record_time, record = parse_params(params)
 
-        # Skip nil record
-        if record.nil?
-          log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
-          if @respond_with_empty_img
-            return RESPONSE_IMG
-          else
-            if @use_204_response
-              return RESPONSE_204
-            else
-              return RESPONSE_200
-            end
+        mes = Fluent::MultiEventStream.new
+        parse_params(params) do |record_time, record|
+          if record.nil?
+            log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
+            next
           end
-        end
 
-        mes = nil
-        # Support batched requests
-        if record.is_a?(Array)
-          mes = Fluent::MultiEventStream.new
-          record.each do |single_record|
-            add_params_to_record(single_record, params)
-
-            if param_time = params['time']
-              param_time = param_time.to_f
-              single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
-            elsif @custom_parser
-              single_time = @custom_parser.parse_time(single_record)
-              single_time, single_record = @custom_parser.convert_values(single_time, single_record)
-            else
-              single_time = convert_time_field(single_record)
-            end
-
-            mes.add(single_time, single_record)
-          end
-        else
           add_params_to_record(record, params)
 
           time = if param_time = params['time']
                    param_time = param_time.to_f
                    param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
                  else
-                   if record_time.nil?
-                     convert_time_field(record)
-                   else
-                     record_time
-                   end
+                   record_time.nil? ? convert_time_field(record) : record_time
                  end
+
+          mes.add(time, record)
         end
       rescue => e
         if @dump_error_log
@@ -261,11 +231,7 @@ module Fluent::Plugin
 
       # TODO server error
       begin
-        if mes
-          router.emit_stream(tag, mes)
-        else
-          router.emit(tag, time, record)
-        end
+        router.emit_stream(tag, mes) unless mes.empty?
       rescue => e
         if @dump_error_log
           log.error "failed to emit data", error: e
@@ -308,20 +274,18 @@ module Fluent::Plugin
     def parse_params_default(params)
       if msgpack = params['msgpack']
         @parser_msgpack.parse(msgpack) do |_time, record|
-          return nil, record
+          yield nil, record
         end
       elsif js = params['json']
         @parser_json.parse(js) do |_time, record|
-          return nil, record
+          yield nil, record
         end
       elsif ndjson = params['ndjson']
-        events = []
         ndjson.split(/\r?\n/).each do |js|
           @parser_json.parse(js) do |_time, record|
-            events.push(record)
+            yield nil, record
           end
         end
-        return nil, events
       else
         raise "'json', 'ndjson' or 'msgpack' parameter is required"
       end
@@ -329,10 +293,9 @@ module Fluent::Plugin
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        @custom_parser.parse(content) { |time, record|
-          raise "Received event is not #{@format_name}: #{content}" if record.nil?
-          return time, record
-        }
+        @custom_parser.parse(content) do |time, record|
+          yield time, record
+        end
       else
         raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
       end
@@ -573,6 +536,8 @@ module Fluent::Plugin
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif /^application\/json/.match?(@content_type)
           params['json'] = @body
+        elsif /^application\/csp-report/.match?(@content_type)
+          params['json'] = @body
         elsif /^application\/msgpack/.match?(@content_type)
           params['msgpack'] = @body
         elsif /^application\/x-ndjson/.match?(@content_type)
@@ -580,7 +545,7 @@ module Fluent::Plugin
         end
         path_info = uri.path
 
-        if (@add_query_params) 
+        if (@add_query_params)
 
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)