fluentd 1.16.4-x86-mingw32 → 1.17.0-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55d9dff2b2c0dc6754261babb59c10b05cb4990b04366a10b6c3ed531e939ef6
-  data.tar.gz: 3fe6adb800c7f359b4dae8ede6c38710549da0bccd6181cc27c95cfa4ec5f776
+  metadata.gz: cc7924012e09f9568cbda96bbb4d1449074a4db721795d74900261b984b04047
+  data.tar.gz: 8c3e8f5ef86237b5c18943ffbf393188338f9dbd3dbcac3d9fb321973ba3b43e
 SHA512:
-  metadata.gz: eed2be6c185fc49df73504535018a7eaf7c5142a5fcebbc08d6e56cba93edc94d1c9ed159456bc52a6925c7e929c8bef4a2f2ca03850798ff52d90054eae849b
-  data.tar.gz: 5c2d5c814e6eeffdbafd831f0fa04ab79dc0f4979c3e3fa35dae941d7cec35c800b735f6c2efafacf91ae265e65d7d1d171ca941d28c60309399f74e61aad554
+  metadata.gz: 7998a2f12349a767e7952dcf3a17a108cfb0f2e2063524862d97b83159012e8b753e46d2e4893e235e6a4bb1d43533f3068dc955601533fddf839f75edcca096
+  data.tar.gz: '09dac9716859404ddb3b563408cab32921c894300f2701b3e7aff89a608be154d0090e321adbbed9e0615459b06a4be3038f3e5b16206e624a0e1e770970c803'

data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml

@@ -0,0 +1,50 @@
+title: "[QA (Japanese)]"
+labels: ["Q&A (Japanese)"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        日本語で気軽に質問するためのカテゴリです。もし他の人が困っているのを見つけたらぜひ回答してあげてください。
+  - type: textarea
+    id: question
+    attributes:
+      label: やりたいこと
+      description: |
+        何について困っているのかを書いてください。試したことや実際の結果を示してください。
+        期待する挙動と実際の結果の違いがあればそれも書くのをおすすめします。
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: 設定した内容
+      description: |
+        どのような設定をして期待する挙動を実現しようとしたのかを書いてください。(例: fluentd.confの内容を貼り付ける)
+      render: apache
+  - type: textarea
+    id: logs
+    attributes:
+      label: ログの内容
+      description: |
+        Fluentdのログを提示してください。エラーログがあると回答の助けになります。(例: fluentd.logの内容を貼り付ける)
+      render: shell
+  - type: textarea
+    id: environment
+    attributes:
+      label: 環境について
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        どんな環境で困っているかの情報がないと、再現できないため誰も回答できないことがあります。
+        必要な情報を記入することをおすすめします。
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Fluent Package version:
+        - Docker image (tag):
+        - Operating system:
+        - Kernel version:
+      render: markdown

data/.github/DISCUSSION_TEMPLATE/q-a.yml

@@ -0,0 +1,47 @@
+title: "[Q&A]"
+labels: ["Q&A"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        It is recommended to support each other.
+  - type: textarea
+    id: question
+    attributes:
+      label: What is a problem?
+      description: |
+        A clear and concise description of what you want to happen.
+        What exactly did you do (or not do) that was effective (or ineffective)?
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Describe the configuration of Fluentd
+      description: |
+        If there is the actual configuration of Fluentd, it will help.
+  - type: textarea
+    id: logs
+    attributes:
+      label: Describe the logs of Fluentd
+      description: |
+        If there are error logs of Fluentd, it will help.
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Please describe your environment information. If will help to support.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Fluent Package version:
+        - Docker image (tag):
+        - Operating system:
+        - Kernel version:
+      render: markdown

data/.github/workflows/test-ruby-head.yml

@@ -0,0 +1,31 @@
+name: Test with Ruby head
+
+on:
+  schedule:
+    - cron: '11 14 * * 0'
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        ruby-version: ['head']
+
+    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install addons
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: sudo apt-get install libgmp3-dev libcap-ng-dev
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: Test
 
 on:
   push:
-    branches: [v1.16]
+    branches: [master]
   pull_request:
-    branches: [v1.16]
+    branches: [master]
 
 jobs:
   test:
@@ -29,4 +29,4 @@ jobs:
       - name: Install dependencies
         run: bundle install
       - name: Run tests
-        run: bundle exec rake test TESTOPTS=-v
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/CHANGELOG.md

@@ -1,5 +1,55 @@
+# v1.17
+
+## Release v1.17.0 - 2024/04/30
+
+### Enhancement
+
+* in_http: Recognize CSP reports as JSON data
+  https://github.com/fluent/fluentd/pull/4282
+* out_http: Add option to reuse connections
+  https://github.com/fluent/fluentd/pull/4330
+* in_tail: Expand glob capability for square brackets and one character matcher
+  https://github.com/fluent/fluentd/pull/4401
+* out_http: Support AWS Signature Version 4 authentication
+  https://github.com/fluent/fluentd/pull/4459
+
+### Bug Fix
+
+* Make sure `parser_json` and `parser_msgpack` return `Hash`.
+  Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`.
+  https://github.com/fluent/fluentd/pull/4474
+* filter_parser: Add error event for multiple parsed results
+  https://github.com/fluent/fluentd/pull/4478
+
+### Misc
+
+* Raise minimum required ruby version
+  https://github.com/fluent/fluentd/pull/4288
+* Require missing dependent gems as of Ruby 3.4-dev
+  https://github.com/fluent/fluentd/pull/4411
+* Minor code refactoring
+  https://github.com/fluent/fluentd/pull/4294
+  https://github.com/fluent/fluentd/pull/4299
+  https://github.com/fluent/fluentd/pull/4302
+  https://github.com/fluent/fluentd/pull/4320
+* CI fixes
+  https://github.com/fluent/fluentd/pull/4369
+  https://github.com/fluent/fluentd/pull/4433
+  https://github.com/fluent/fluentd/pull/4452
+  https://github.com/fluent/fluentd/pull/4477
+* github: unify YAML file extension to .yml
+  https://github.com/fluent/fluentd/pull/4429
+
 # v1.16
 
+## Release v1.16.5 - 2024/03/27
+
+### Bug Fix
+
+* Buffer: Fix emit error of v1.16.4 sometimes failing to process large data
+  exceeding chunk size limit
+  https://github.com/fluent/fluentd/pull/4447
+
 ## Release v1.16.4 - 2024/03/14
 
 ### Bug Fix

data/README.md

@@ -29,7 +29,7 @@ Fluentd: Open-Source Log Collector
 
 ### Prerequisites
 
-- Ruby 2.4 or later
+- Ruby 2.7 or later
 - git
 
 `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.

data/Rakefile

@@ -13,7 +13,7 @@ task test: [:base_test]
 namespace :build do
   desc 'Build gems for all platforms'
   task :all do
-    Bundler.with_clean_env do
+    Bundler.with_original_env do
       %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
         ENV['GEM_BUILD_FAKE_PLATFORM'] = name
         Rake::Task["build"].execute

data/fluentd.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 2.4'
+  gem.required_ruby_version = '>= 2.7'
 
   gem.add_runtime_dependency("bundler")
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
@@ -30,6 +30,11 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", ["~> 1.4"])
 
+  # gems that aren't default gems as of Ruby 3.4
+  gem.add_runtime_dependency("base64", ["~> 0.2"])
+  gem.add_runtime_dependency("csv", ["~> 3.2"])
+  gem.add_runtime_dependency("drb", ["~> 2.2"])
+
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
@@ -51,4 +56,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
   gem.add_development_dependency("async", "~> 1.23")
   gem.add_development_dependency("async-http", ">= 0.50.0")
+  gem.add_development_dependency("aws-sigv4", ["~> 1.8"])
+  gem.add_development_dependency("aws-sdk-core", ["~> 3.191"])
+  gem.add_development_dependency("rexml", ["~> 3.2"])
 end

data/lib/fluent/command/binlog_reader.rb

@@ -130,7 +130,7 @@ module BinlogReaderCommand
     private
 
     def configure_option_parser
-      @options.merge!(config_params: {})
+      @options[:config_params] = {}
 
       @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
 

data/lib/fluent/config/configure_proxy.rb

@@ -201,7 +201,7 @@ module Fluent
 
       def overwrite_defaults(other) # other is owner plugin's corresponding proxy
         self.defaults = self.defaults.merge(other.defaults)
-        self.sections.keys.each do |section_key|
+        self.sections.each_key do |section_key|
           if other.sections.has_key?(section_key)
             self.sections[section_key].overwrite_defaults(other.sections[section_key])
           end
@@ -274,7 +274,7 @@ module Fluent
         option_value_type!(name, opts, :deprecated, String)
         option_value_type!(name, opts, :obsoleted, String)
         if type == :enum
-          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
+          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?(Symbol)
             raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
           end
         end

data/lib/fluent/config/types.rb

@@ -123,7 +123,7 @@ module Fluent
 
       s = val.to_sym
       list = opts[:list]
-      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
+      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?(Symbol)
       unless list.include?(s)
         raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
       end

data/lib/fluent/configurable.rb

@@ -31,13 +31,13 @@ module Fluent
       super
       # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
       proxy = self.class.merged_configure_proxy
-      proxy.params.keys.each do |name|
+      proxy.params.each_key do |name|
         next if name.to_s.start_with?('@')
         if proxy.defaults.has_key?(name)
           instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
         end
       end
-      proxy.sections.keys.each do |name|
+      proxy.sections.each_key do |name|
         next if name.to_s.start_with?('@')
         subproxy = proxy.sections[name]
         if subproxy.multi?

data/lib/fluent/counter/mutex_hash.rb

@@ -54,7 +54,7 @@ module Fluent
               if mutex.try_lock
                 locks[key] = mutex
               else
-                locks.values.each(&:unlock)
+                locks.each_value(&:unlock)
                 locks = {}          # flush locked keys
                 break
               end

data/lib/fluent/fluent_log_event_router.rb

@@ -47,8 +47,6 @@ module Fluent
         # it's not suppressed in default event router for non-log-event events
         log_event_router.suppress_missing_match!
 
-        log_event_router = log_event_router
-
         unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
         unless unmatched_tags.empty?
           $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags

data/lib/fluent/plugin/buf_file.rb

@@ -56,7 +56,7 @@ module Fluent
 
         @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
 
-        multi_workers_configured = owner.system_config.workers > 1 ? true : false
+        multi_workers_configured = owner.system_config.workers > 1
 
         using_plugin_root_dir = false
         unless @path

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -399,7 +399,7 @@ module Fluent
           end
 
           if chunk.slice(0, 2) == BUFFER_HEADER
-            size = chunk.slice(2, 4).unpack('N').first
+            size = chunk.slice(2, 4).unpack1('N')
             if size
               return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
             end

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -238,17 +238,16 @@ module Fluent
           callback.call(file) if callback
         end
 
-        URI_PARSER = URI::Parser.new
         ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
 
         def encode_key(metadata)
           k = @key ? metadata.variables[@key] : metadata.tag
           k ||= ''
-          URI_PARSER.escape(k, ESCAPE_REGEXP)
+          URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
         end
 
         def decode_key(key)
-          URI_PARSER.unescape(key)
+          URI::DEFAULT_PARSER.unescape(key)
         end
 
         def create_new_chunk(path, metadata, perm)

data/lib/fluent/plugin/buffer.rb

@@ -764,94 +764,95 @@ module Fluent
         while writing_splits_index < splits.size
           chunk = get_next_chunk.call
           errors = []
+          # The chunk must be locked until being passed to &block.
+          chunk.mon_enter
           modified_chunks << {chunk: chunk, adding_bytesize: 0, errors: errors}
-          chunk.synchronize do
-            raise ShouldRetry unless chunk.writable?
-            staged_chunk_used = true if chunk.staged?
-
-            original_bytesize = committed_bytesize = chunk.bytesize
-            begin
-              while writing_splits_index < splits.size
-                split = splits[writing_splits_index]
-                formatted_split = format ? format.call(split) : nil
 
-                if split.size == 1 # Check BufferChunkOverflowError
-                  determined_bytesize = nil
-                  if @compress != :text
-                    determined_bytesize = nil
-                  elsif formatted_split
-                    determined_bytesize = formatted_split.bytesize
-                  elsif split.first.respond_to?(:bytesize)
-                    determined_bytesize = split.first.bytesize
-                  end
+          raise ShouldRetry unless chunk.writable?
+          staged_chunk_used = true if chunk.staged?
 
-                  if determined_bytesize && determined_bytesize > @chunk_limit_size
-                    # It is a obvious case that BufferChunkOverflowError should be raised here.
-                    # But if it raises here, already processed 'split' or
-                    # the proceeding 'split' will be lost completely.
-                    # So it is a last resort to delay raising such a exception
-                    errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                    writing_splits_index += 1
-                    next
-                  end
+          original_bytesize = committed_bytesize = chunk.bytesize
+          begin
+            while writing_splits_index < splits.size
+              split = splits[writing_splits_index]
+              formatted_split = format ? format.call(split) : nil
 
-                  if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
-                    # The split will (might) cause size over so keep already processed
-                    # 'split' content here (allow performance regression a bit).
-                    chunk.commit
-                    committed_bytesize = chunk.bytesize
-                  end
+              if split.size == 1 # Check BufferChunkOverflowError
+                determined_bytesize = nil
+                if @compress != :text
+                  determined_bytesize = nil
+                elsif formatted_split
+                  determined_bytesize = formatted_split.bytesize
+                elsif split.first.respond_to?(:bytesize)
+                  determined_bytesize = split.first.bytesize
                 end
 
-                if format
-                  chunk.concat(formatted_split, split.size)
-                else
-                  chunk.append(split, compress: @compress)
+                if determined_bytesize && determined_bytesize > @chunk_limit_size
+                  # It is a obvious case that BufferChunkOverflowError should be raised here.
+                  # But if it raises here, already processed 'split' or
+                  # the proceeding 'split' will be lost completely.
+                  # So it is a last resort to delay raising such a exception
+                  errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                  writing_splits_index += 1
+                  next
                 end
-                adding_bytes = chunk.bytesize - committed_bytesize
 
-                if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
-                  chunk.rollback
+                if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
+                  # The split will (might) cause size over so keep already processed
+                  # 'split' content here (allow performance regression a bit).
+                  chunk.commit
                   committed_bytesize = chunk.bytesize
+                end
+              end
 
-                  if split.size == 1 # Check BufferChunkOverflowError again
-                    if adding_bytes > @chunk_limit_size
-                      errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                      writing_splits_index += 1
-                      next
-                    else
-                      # As already processed content is kept after rollback, then unstaged chunk should be queued.
-                      # After that, re-process current split again.
-                      # New chunk should be allocated, to do it, modify @stage and so on.
-                      synchronize { @stage.delete(modified_metadata) }
-                      staged_chunk_used = false
-                      chunk.unstaged!
-                      break
-                    end
-                  end
+              if format
+                chunk.concat(formatted_split, split.size)
+              else
+                chunk.append(split, compress: @compress)
+              end
+              adding_bytes = chunk.bytesize - committed_bytesize
 
-                  if chunk_size_full?(chunk) || split.size == 1
-                    enqueue_chunk_before_retry = true
+              if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
+                chunk.rollback
+                committed_bytesize = chunk.bytesize
+
+                if split.size == 1 # Check BufferChunkOverflowError again
+                  if adding_bytes > @chunk_limit_size
+                    errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                    writing_splits_index += 1
+                    next
                   else
-                    splits_count *= 10
+                    # As already processed content is kept after rollback, then unstaged chunk should be queued.
+                    # After that, re-process current split again.
+                    # New chunk should be allocated, to do it, modify @stage and so on.
+                    synchronize { @stage.delete(modified_metadata) }
+                    staged_chunk_used = false
+                    chunk.unstaged!
+                    break
                   end
+                end
 
-                  raise ShouldRetry
+                if chunk_size_full?(chunk) || split.size == 1
+                  enqueue_chunk_before_retry = true
+                else
+                  splits_count *= 10
                 end
 
-                writing_splits_index += 1
+                raise ShouldRetry
+              end
 
-                if chunk_size_full?(chunk)
-                  break
-                end
+              writing_splits_index += 1
+
+              if chunk_size_full?(chunk)
+                break
               end
-            rescue
-              chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
-              raise
             end
-
-            modified_chunks.last[:adding_bytesize] = chunk.bytesize - original_bytesize
+          rescue
+            chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
+            raise
           end
+
+          modified_chunks.last[:adding_bytesize] = chunk.bytesize - original_bytesize
         end
         modified_chunks.each do |data|
           block.call(data[:chunk], data[:adding_bytesize], data[:errors])
@@ -863,9 +864,15 @@ module Fluent
           if chunk.unstaged?
             chunk.purge rescue nil
           end
+          chunk.mon_exit rescue nil
         end
         enqueue_chunk(metadata) if enqueue_chunk_before_retry
         retry
+      ensure
+        modified_chunks.each do |data|
+          chunk = data[:chunk]
+          chunk.mon_exit
+        end
       end
 
       STATS_KEYS = [

data/lib/fluent/plugin/filter_parser.rb

@@ -70,6 +70,13 @@ module Fluent::Plugin
         end
       end
       begin
+        # Note: https://github.com/fluent/fluentd/issues/4100
+        # If the parser returns multiple records from one raw_value,
+        # this returns only the first one record.
+        # This should be fixed in the future version.
+        result_time = nil
+        result_record = nil
+
         @parser.parse(raw_value) do |t, values|
           if values
             t = if @reserve_time
@@ -79,20 +86,31 @@ module Fluent::Plugin
                 end
             @accessor.delete(record) if @remove_key_name_field
             r = handle_parsed(tag, record, t, values)
-            return t, r
+
+            if result_record.nil?
+              result_time = t
+              result_record = r
+            else
+              if @emit_invalid_record_to_error
+                router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
+                  "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
+                ))
+              end
+            end
           else
             if @emit_invalid_record_to_error
               router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
             end
-            if @reserve_data
-              t = time
-              r = handle_parsed(tag, record, time, {})
-              return t, r
-            else
-              return FAILED_RESULT
-            end
+
+            next unless @reserve_data
+            next unless result_record.nil?
+
+            result_time = time
+            result_record = handle_parsed(tag, record, time, {})
           end
         end
+
+        return result_time, result_record
       rescue Fluent::Plugin::Parser::ParserError => e
         if @emit_invalid_record_to_error
           raise e