fluentd 1.16.5-x86-mingw32 → 1.17.0-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b91c47ef9342fa99cc377c8bafc37afa3b67511ba4b946a15a95dc2e845415f4
-  data.tar.gz: 402b8c065ab503c337ff90dcd3bcbf08a79fd93df2b5de98f155588060dc5840
+  metadata.gz: cc7924012e09f9568cbda96bbb4d1449074a4db721795d74900261b984b04047
+  data.tar.gz: 8c3e8f5ef86237b5c18943ffbf393188338f9dbd3dbcac3d9fb321973ba3b43e
 SHA512:
-  metadata.gz: 6e9b7f9234357fb1ef811dd6087224221dba7e1030aa448eaaf70cee1f7867297ad6bb208ff9e6d2ccfc82a4f4fd6584e137a2b8aa765ca99d3a3054ea2dda9b
-  data.tar.gz: 9040b8269bd7c3c5be178340f0d264a82affacba575fb1682d67d9088d66ab2eb617cf24baf94e949f98a240c12e3ac58fbab3981f31bcf41aab2d567de5a290
+  metadata.gz: 7998a2f12349a767e7952dcf3a17a108cfb0f2e2063524862d97b83159012e8b753e46d2e4893e235e6a4bb1d43533f3068dc955601533fddf839f75edcca096
+  data.tar.gz: '09dac9716859404ddb3b563408cab32921c894300f2701b3e7aff89a608be154d0090e321adbbed9e0615459b06a4be3038f3e5b16206e624a0e1e770970c803'

data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml

@@ -0,0 +1,50 @@
+title: "[QA (Japanese)]"
+labels: ["Q&A (Japanese)"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        日本語で気軽に質問するためのカテゴリです。もし他の人が困っているのを見つけたらぜひ回答してあげてください。
+  - type: textarea
+    id: question
+    attributes:
+      label: やりたいこと
+      description: |
+        何について困っているのかを書いてください。試したことや実際の結果を示してください。
+        期待する挙動と実際の結果の違いがあればそれも書くのをおすすめします。
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: 設定した内容
+      description: |
+        どのような設定をして期待する挙動を実現しようとしたのかを書いてください。(例: fluentd.confの内容を貼り付ける)
+      render: apache
+  - type: textarea
+    id: logs
+    attributes:
+      label: ログの内容
+      description: |
+        Fluentdのログを提示してください。エラーログがあると回答の助けになります。(例: fluentd.logの内容を貼り付ける)
+      render: shell
+  - type: textarea
+    id: environment
+    attributes:
+      label: 環境について
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        どんな環境で困っているかの情報がないと、再現できないため誰も回答できないことがあります。
+        必要な情報を記入することをおすすめします。
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Fluent Package version:
+        - Docker image (tag):
+        - Operating system:
+        - Kernel version:
+      render: markdown

data/.github/DISCUSSION_TEMPLATE/q-a.yml

@@ -0,0 +1,47 @@
+title: "[Q&A]"
+labels: ["Q&A"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        It is recommended to support each other.
+  - type: textarea
+    id: question
+    attributes:
+      label: What is a problem?
+      description: |
+        A clear and concise description of what you want to happen.
+        What exactly did you do (or not do) that was effective (or ineffective)?
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Describe the configuration of Fluentd
+      description: |
+        If there is the actual configuration of Fluentd, it will help.
+  - type: textarea
+    id: logs
+    attributes:
+      label: Describe the logs of Fluentd
+      description: |
+        If there are error logs of Fluentd, it will help.
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Please describe your environment information. If will help to support.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Fluent Package version:
+        - Docker image (tag):
+        - Operating system:
+        - Kernel version:
+      render: markdown

data/.github/workflows/test-ruby-head.yml

@@ -0,0 +1,31 @@
+name: Test with Ruby head
+
+on:
+  schedule:
+    - cron: '11 14 * * 0'
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        ruby-version: ['head']
+
+    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install addons
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: sudo apt-get install libgmp3-dev libcap-ng-dev
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: Test
 
 on:
   push:
-    branches: [v1.16]
+    branches: [master]
   pull_request:
-    branches: [v1.16]
+    branches: [master]
 
 jobs:
   test:
@@ -29,4 +29,4 @@ jobs:
       - name: Install dependencies
         run: bundle install
       - name: Run tests
-        run: bundle exec rake test TESTOPTS=-v
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/CHANGELOG.md

@@ -1,3 +1,45 @@
+# v1.17
+
+## Release v1.17.0 - 2024/04/30
+
+### Enhancement
+
+* in_http: Recognize CSP reports as JSON data
+  https://github.com/fluent/fluentd/pull/4282
+* out_http: Add option to reuse connections
+  https://github.com/fluent/fluentd/pull/4330
+* in_tail: Expand glob capability for square brackets and one character matcher
+  https://github.com/fluent/fluentd/pull/4401
+* out_http: Support AWS Signature Version 4 authentication
+  https://github.com/fluent/fluentd/pull/4459
+
+### Bug Fix
+
+* Make sure `parser_json` and `parser_msgpack` return `Hash`.
+  Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`.
+  https://github.com/fluent/fluentd/pull/4474
+* filter_parser: Add error event for multiple parsed results
+  https://github.com/fluent/fluentd/pull/4478
+
+### Misc
+
+* Raise minimum required ruby version
+  https://github.com/fluent/fluentd/pull/4288
+* Require missing dependent gems as of Ruby 3.4-dev
+  https://github.com/fluent/fluentd/pull/4411
+* Minor code refactoring
+  https://github.com/fluent/fluentd/pull/4294
+  https://github.com/fluent/fluentd/pull/4299
+  https://github.com/fluent/fluentd/pull/4302
+  https://github.com/fluent/fluentd/pull/4320
+* CI fixes
+  https://github.com/fluent/fluentd/pull/4369
+  https://github.com/fluent/fluentd/pull/4433
+  https://github.com/fluent/fluentd/pull/4452
+  https://github.com/fluent/fluentd/pull/4477
+* github: unify YAML file extension to .yml
+  https://github.com/fluent/fluentd/pull/4429
+
 # v1.16
 
 ## Release v1.16.5 - 2024/03/27

data/README.md

@@ -29,7 +29,7 @@ Fluentd: Open-Source Log Collector
 
 ### Prerequisites
 
-- Ruby 2.4 or later
+- Ruby 2.7 or later
 - git
 
 `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.

data/Rakefile

@@ -13,7 +13,7 @@ task test: [:base_test]
 namespace :build do
   desc 'Build gems for all platforms'
   task :all do
-    Bundler.with_clean_env do
+    Bundler.with_original_env do
       %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
         ENV['GEM_BUILD_FAKE_PLATFORM'] = name
         Rake::Task["build"].execute

data/fluentd.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 2.4'
+  gem.required_ruby_version = '>= 2.7'
 
   gem.add_runtime_dependency("bundler")
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
@@ -30,6 +30,11 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", ["~> 1.4"])
 
+  # gems that aren't default gems as of Ruby 3.4
+  gem.add_runtime_dependency("base64", ["~> 0.2"])
+  gem.add_runtime_dependency("csv", ["~> 3.2"])
+  gem.add_runtime_dependency("drb", ["~> 2.2"])
+
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
@@ -51,4 +56,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
   gem.add_development_dependency("async", "~> 1.23")
   gem.add_development_dependency("async-http", ">= 0.50.0")
+  gem.add_development_dependency("aws-sigv4", ["~> 1.8"])
+  gem.add_development_dependency("aws-sdk-core", ["~> 3.191"])
+  gem.add_development_dependency("rexml", ["~> 3.2"])
 end

data/lib/fluent/command/binlog_reader.rb

@@ -130,7 +130,7 @@ module BinlogReaderCommand
     private
 
     def configure_option_parser
-      @options.merge!(config_params: {})
+      @options[:config_params] = {}
 
       @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
 

data/lib/fluent/config/configure_proxy.rb

@@ -201,7 +201,7 @@ module Fluent
 
       def overwrite_defaults(other) # other is owner plugin's corresponding proxy
         self.defaults = self.defaults.merge(other.defaults)
-        self.sections.keys.each do |section_key|
+        self.sections.each_key do |section_key|
           if other.sections.has_key?(section_key)
             self.sections[section_key].overwrite_defaults(other.sections[section_key])
           end
@@ -274,7 +274,7 @@ module Fluent
         option_value_type!(name, opts, :deprecated, String)
         option_value_type!(name, opts, :obsoleted, String)
         if type == :enum
-          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
+          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?(Symbol)
             raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
           end
         end

data/lib/fluent/config/types.rb

@@ -123,7 +123,7 @@ module Fluent
 
       s = val.to_sym
       list = opts[:list]
-      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
+      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?(Symbol)
       unless list.include?(s)
         raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
       end

data/lib/fluent/configurable.rb

@@ -31,13 +31,13 @@ module Fluent
       super
       # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
       proxy = self.class.merged_configure_proxy
-      proxy.params.keys.each do |name|
+      proxy.params.each_key do |name|
         next if name.to_s.start_with?('@')
         if proxy.defaults.has_key?(name)
           instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
         end
       end
-      proxy.sections.keys.each do |name|
+      proxy.sections.each_key do |name|
         next if name.to_s.start_with?('@')
         subproxy = proxy.sections[name]
         if subproxy.multi?

data/lib/fluent/counter/mutex_hash.rb

@@ -54,7 +54,7 @@ module Fluent
               if mutex.try_lock
                 locks[key] = mutex
               else
-                locks.values.each(&:unlock)
+                locks.each_value(&:unlock)
                 locks = {}          # flush locked keys
                 break
               end

data/lib/fluent/fluent_log_event_router.rb

@@ -47,8 +47,6 @@ module Fluent
         # it's not suppressed in default event router for non-log-event events
         log_event_router.suppress_missing_match!
 
-        log_event_router = log_event_router
-
         unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
         unless unmatched_tags.empty?
           $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags

data/lib/fluent/plugin/buf_file.rb

@@ -56,7 +56,7 @@ module Fluent
 
         @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
 
-        multi_workers_configured = owner.system_config.workers > 1 ? true : false
+        multi_workers_configured = owner.system_config.workers > 1
 
         using_plugin_root_dir = false
         unless @path

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -399,7 +399,7 @@ module Fluent
           end
 
           if chunk.slice(0, 2) == BUFFER_HEADER
-            size = chunk.slice(2, 4).unpack('N').first
+            size = chunk.slice(2, 4).unpack1('N')
             if size
               return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
             end

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -238,17 +238,16 @@ module Fluent
           callback.call(file) if callback
         end
 
-        URI_PARSER = URI::Parser.new
         ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
 
         def encode_key(metadata)
           k = @key ? metadata.variables[@key] : metadata.tag
           k ||= ''
-          URI_PARSER.escape(k, ESCAPE_REGEXP)
+          URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
         end
 
         def decode_key(key)
-          URI_PARSER.unescape(key)
+          URI::DEFAULT_PARSER.unescape(key)
         end
 
         def create_new_chunk(path, metadata, perm)

data/lib/fluent/plugin/filter_parser.rb

@@ -70,6 +70,13 @@ module Fluent::Plugin
         end
       end
       begin
+        # Note: https://github.com/fluent/fluentd/issues/4100
+        # If the parser returns multiple records from one raw_value,
+        # this returns only the first one record.
+        # This should be fixed in the future version.
+        result_time = nil
+        result_record = nil
+
         @parser.parse(raw_value) do |t, values|
           if values
             t = if @reserve_time
@@ -79,20 +86,31 @@ module Fluent::Plugin
                 end
             @accessor.delete(record) if @remove_key_name_field
             r = handle_parsed(tag, record, t, values)
-            return t, r
+
+            if result_record.nil?
+              result_time = t
+              result_record = r
+            else
+              if @emit_invalid_record_to_error
+                router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
+                  "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
+                ))
+              end
+            end
           else
             if @emit_invalid_record_to_error
               router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
             end
-            if @reserve_data
-              t = time
-              r = handle_parsed(tag, record, time, {})
-              return t, r
-            else
-              return FAILED_RESULT
-            end
+
+            next unless @reserve_data
+            next unless result_record.nil?
+
+            result_time = time
+            result_record = handle_parsed(tag, record, time, {})
           end
         end
+
+        return result_time, result_record
       rescue Fluent::Plugin::Parser::ParserError => e
         if @emit_invalid_record_to_error
           raise e

data/lib/fluent/plugin/in_http.rb

@@ -203,54 +203,24 @@ module Fluent::Plugin
       begin
         path = path_info[1..-1]  # remove /
         tag = path.split('/').join('.')
-        record_time, record = parse_params(params)
 
-        # Skip nil record
-        if record.nil?
-          log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
-          if @respond_with_empty_img
-            return RESPONSE_IMG
-          else
-            if @use_204_response
-              return RESPONSE_204
-            else
-              return RESPONSE_200
-            end
+        mes = Fluent::MultiEventStream.new
+        parse_params(params) do |record_time, record|
+          if record.nil?
+            log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
+            next
           end
-        end
 
-        mes = nil
-        # Support batched requests
-        if record.is_a?(Array)
-          mes = Fluent::MultiEventStream.new
-          record.each do |single_record|
-            add_params_to_record(single_record, params)
-
-            if param_time = params['time']
-              param_time = param_time.to_f
-              single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
-            elsif @custom_parser
-              single_time = @custom_parser.parse_time(single_record)
-              single_time, single_record = @custom_parser.convert_values(single_time, single_record)
-            else
-              single_time = convert_time_field(single_record)
-            end
-
-            mes.add(single_time, single_record)
-          end
-        else
           add_params_to_record(record, params)
 
           time = if param_time = params['time']
                    param_time = param_time.to_f
                    param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
                  else
-                   if record_time.nil?
-                     convert_time_field(record)
-                   else
-                     record_time
-                   end
+                   record_time.nil? ? convert_time_field(record) : record_time
                  end
+
+          mes.add(time, record)
         end
       rescue => e
         if @dump_error_log
@@ -261,11 +231,7 @@ module Fluent::Plugin
 
       # TODO server error
       begin
-        if mes
-          router.emit_stream(tag, mes)
-        else
-          router.emit(tag, time, record)
-        end
+        router.emit_stream(tag, mes) unless mes.empty?
       rescue => e
         if @dump_error_log
           log.error "failed to emit data", error: e
@@ -308,20 +274,18 @@ module Fluent::Plugin
     def parse_params_default(params)
       if msgpack = params['msgpack']
         @parser_msgpack.parse(msgpack) do |_time, record|
-          return nil, record
+          yield nil, record
         end
       elsif js = params['json']
         @parser_json.parse(js) do |_time, record|
-          return nil, record
+          yield nil, record
         end
       elsif ndjson = params['ndjson']
-        events = []
         ndjson.split(/\r?\n/).each do |js|
           @parser_json.parse(js) do |_time, record|
-            events.push(record)
+            yield nil, record
           end
         end
-        return nil, events
       else
         raise "'json', 'ndjson' or 'msgpack' parameter is required"
       end
@@ -329,10 +293,9 @@ module Fluent::Plugin
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        @custom_parser.parse(content) { |time, record|
-          raise "Received event is not #{@format_name}: #{content}" if record.nil?
-          return time, record
-        }
+        @custom_parser.parse(content) do |time, record|
+          yield time, record
+        end
       else
         raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
       end
@@ -573,6 +536,8 @@ module Fluent::Plugin
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif /^application\/json/.match?(@content_type)
           params['json'] = @body
+        elsif /^application\/csp-report/.match?(@content_type)
+          params['json'] = @body
         elsif /^application\/msgpack/.match?(@content_type)
           params['msgpack'] = @body
         elsif /^application\/x-ndjson/.match?(@content_type)
@@ -580,7 +545,7 @@ module Fluent::Plugin
         end
         path_info = uri.path
 
-        if (@add_query_params) 
+        if (@add_query_params)
 
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)
 

data/lib/fluent/plugin/in_tail.rb

@@ -65,6 +65,8 @@ module Fluent::Plugin
     config_param :path, :string
     desc 'path delimiter used for spliting path config'
     config_param :path_delimiter, :string, default: ','
+    desc 'Choose using glob patterns. Adding capabilities to handle [] and ?, and {}.'
+    config_param :glob_policy, :enum, list: [:backward_compatible, :extended, :always], default: :backward_compatible
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
@@ -141,6 +143,14 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
       end
 
+      if @glob_policy == :always && @path_delimiter == ','
+        raise Fluent::ConfigError, "cannot use glob_policy as always with the default path_delimitor: `,\""
+      end
+
+      if @glob_policy == :extended && /\{.*,.*\}/.match(@path) && extended_glob_pattern(@path)
+        raise Fluent::ConfigError, "cannot include curly braces with glob patterns in `#{@path}\". Use glob_policy always instead."
+      end
+
       if RESERVED_CHARS.include?(@path_delimiter)
         rc = RESERVED_CHARS.join(', ')
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
@@ -288,6 +298,28 @@ module Fluent::Plugin
         @capability.have_capability?(:effective, :dac_override)
     end
 
+    def extended_glob_pattern(path)
+      path.include?('*') || path.include?('?') || /\[.*\]/.match(path)
+    end
+
+    # Curly braces is not supported with default path_delimiter
+    # because the default delimiter of path is ",".
+    # This should be collided for wildcard pattern for curly braces and
+    # be handled as an error on #configure.
+    def use_glob?(path)
+      if @glob_policy == :always
+        # For future extensions, we decided to use `always' term to handle
+        # regular expressions as much as possible.
+        # This is because not using `true' as a returning value
+        # when choosing :always here.
+        extended_glob_pattern(path) || /\{.*,.*\}/.match(path)
+      elsif @glob_policy == :extended
+        extended_glob_pattern(path)
+      elsif @glob_policy == :backward_compatible
+        path.include?('*')
+      end
+    end
+
     def expand_paths
       date = Fluent::EventTime.now
       paths = []
@@ -297,7 +329,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        if path.include?('*')
+        if use_glob?(path)
           paths += Dir.glob(path).select { |p|
             begin
               is_file = !File.directory?(p)
@@ -332,7 +364,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        path.include?('*') ? Dir.glob(path) : path
+        use_glob?(path) ? Dir.glob(path) : path
       }.flatten.uniq
       # filter out non existing files, so in case pattern is without '*' we don't do unnecessary work
       hash = {}