fluentd 1.16.3 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8a64e2943ee3f6abc8c90018a85205b0bd54a8fc5f367da3d5e9ce53f03db4f
-  data.tar.gz: 0d06e0bec37b228eea16a92353a70b0198a9f6b3fef88627898f6ef16ae2322b
+  metadata.gz: df7f40114152fe9cd139ef2c195e7baf19b501a3e949ba8412de445a3337c26f
+  data.tar.gz: e7b728c294b2797223405571ffc66853fb21210fe696b5299179c8a85f1c51a0
 SHA512:
-  metadata.gz: 90e357ccd0f4c02013739c538585d3a1b79f3f3d423efc4bd668496e7b318b87358e3bc0618393b85ce07885445734dadbdc4112a7cd082be4aa89d738d94066
-  data.tar.gz: bfa4f0817153e79538b24af9af0a840cd6ecd6a8ecc2a0a1536a5268b7d29aae7671371181050dac4ba670538d9eb40efbbd9634936217caf7b3ecec090b67e1
+  metadata.gz: ad48b5478ae93b0c41e6db5cec80b1e67ac18c24b44ce43193143150d8fbf765b3b2674ddc6e32dfd649095647bb6113687c436fa463249f0e1ab64c665ab7b3
+  data.tar.gz: 4cfc5bb3ecd1b116346d6d899c243142c4c4a8768e980f895a3d5b6ee3a49b5b7fc132710316a657fb6353651b7fe42345371d536ed6123d52bce3a79bd4ff35

data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml

@@ -0,0 +1,50 @@
+title: "[QA (Japanese)]"
+labels: ["Q&A (Japanese)"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        日本語で気軽に質問するためのカテゴリです。もし他の人が困っているのを見つけたらぜひ回答してあげてください。
+  - type: textarea
+    id: question
+    attributes:
+      label: やりたいこと
+      description: |
+        何について困っているのかを書いてください。試したことや実際の結果を示してください。
+        期待する挙動と実際の結果の違いがあればそれも書くのをおすすめします。
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: 設定した内容
+      description: |
+        どのような設定をして期待する挙動を実現しようとしたのかを書いてください。(例: fluentd.confの内容を貼り付ける)
+      render: apache
+  - type: textarea
+    id: logs
+    attributes:
+      label: ログの内容
+      description: |
+        Fluentdのログを提示してください。エラーログがあると回答の助けになります。(例: fluentd.logの内容を貼り付ける)
+      render: shell
+  - type: textarea
+    id: environment
+    attributes:
+      label: 環境について
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        どんな環境で困っているかの情報がないと、再現できないため誰も回答できないことがあります。
+        必要な情報を記入することをおすすめします。
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Fluent Package version:
+        - Docker image (tag):
+        - Operating system:
+        - Kernel version:
+      render: markdown

data/.github/DISCUSSION_TEMPLATE/q-a.yml

@@ -0,0 +1,47 @@
+title: "[Q&A]"
+labels: ["Q&A"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        It is recommended to support each other.
+  - type: textarea
+    id: question
+    attributes:
+      label: What is a problem?
+      description: |
+        A clear and concise description of what you want to happen.
+        What exactly did you do (or not do) that was effective (or ineffective)?
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Describe the configuration of Fluentd
+      description: |
+        If there is the actual configuration of Fluentd, it will help.
+  - type: textarea
+    id: logs
+    attributes:
+      label: Describe the logs of Fluentd
+      description: |
+        If there are error logs of Fluentd, it will help.
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Please describe your environment information. If will help to support.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Fluent Package version:
+        - Docker image (tag):
+        - Operating system:
+        - Kernel version:
+      render: markdown

data/.github/workflows/test-ruby-head.yml

@@ -0,0 +1,31 @@
+name: Test with Ruby head
+
+on:
+  schedule:
+    - cron: '11 14 * * 0'
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        ruby-version: ['head']
+
+    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install addons
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: sudo apt-get install libgmp3-dev libcap-ng-dev
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/.github/workflows/{linux-test.yaml → test.yml}

@@ -1,27 +1,22 @@
-name: Testing on Ubuntu
+name: Test
 
 on:
   push:
-    branches: [master, v1.16]
+    branches: [master]
   pull_request:
-    branches: [master, v1.16]
+    branches: [master]
 
 jobs:
   test:
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
+    continue-on-error: false
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['3.2', '3.1', '3.0', '2.7']
-        os: [ubuntu-latest]
-        experimental: [false]
-        include:
-          - ruby-version: head
-            os: ubuntu-latest
-            experimental: true
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        ruby-version: ['3.3', '3.2', '3.1', '3.0', '2.7']
 
-    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
+    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
       - name: Set up Ruby
@@ -29,8 +24,9 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby-version }}
       - name: Install addons
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: sudo apt-get install libgmp3-dev libcap-ng-dev
       - name: Install dependencies
         run: bundle install
       - name: Run tests
-        run: bundle exec rake test
+        run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"

data/CHANGELOG.md

@@ -1,5 +1,71 @@
+# v1.17
+
+## Release v1.17.0 - 2024/04/30
+
+### Enhancement
+
+* in_http: Recognize CSP reports as JSON data
+  https://github.com/fluent/fluentd/pull/4282
+* out_http: Add option to reuse connections
+  https://github.com/fluent/fluentd/pull/4330
+* in_tail: Expand glob capability for square brackets and one character matcher
+  https://github.com/fluent/fluentd/pull/4401
+* out_http: Support AWS Signature Version 4 authentication
+  https://github.com/fluent/fluentd/pull/4459
+
+### Bug Fix
+
+* Make sure `parser_json` and `parser_msgpack` return `Hash`.
+  Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`.
+  https://github.com/fluent/fluentd/pull/4474
+* filter_parser: Add error event for multiple parsed results
+  https://github.com/fluent/fluentd/pull/4478
+
+### Misc
+
+* Raise minimum required ruby version
+  https://github.com/fluent/fluentd/pull/4288
+* Require missing dependent gems as of Ruby 3.4-dev
+  https://github.com/fluent/fluentd/pull/4411
+* Minor code refactoring
+  https://github.com/fluent/fluentd/pull/4294
+  https://github.com/fluent/fluentd/pull/4299
+  https://github.com/fluent/fluentd/pull/4302
+  https://github.com/fluent/fluentd/pull/4320
+* CI fixes
+  https://github.com/fluent/fluentd/pull/4369
+  https://github.com/fluent/fluentd/pull/4433
+  https://github.com/fluent/fluentd/pull/4452
+  https://github.com/fluent/fluentd/pull/4477
+* github: unify YAML file extension to .yml
+  https://github.com/fluent/fluentd/pull/4429
+
 # v1.16
 
+## Release v1.16.5 - 2024/03/27
+
+### Bug Fix
+
+* Buffer: Fix emit error of v1.16.4 sometimes failing to process large data
+  exceeding chunk size limit
+  https://github.com/fluent/fluentd/pull/4447
+
+## Release v1.16.4 - 2024/03/14
+
+### Bug Fix
+
+* Fix to avoid processing discarded chunks in write_step_by_step.
+  It fixes not to raise pile of IOError when many `chunk
+  bytes limit exceeds` errors are occurred.
+  https://github.com/fluent/fluentd/pull/4342
+* in_tail: Fix tail watchers in `rotate_wait` state not being managed.
+  https://github.com/fluent/fluentd/pull/4334
+
+### Misc
+
+* buffer: Avoid unnecessary log processing. It will improve performance.
+  https://github.com/fluent/fluentd/pull/4331
+
 ## Release v1.16.3 - 2023/11/14
 
 ### Bug Fix

data/README.md

@@ -29,7 +29,7 @@ Fluentd: Open-Source Log Collector
 
 ### Prerequisites
 
-- Ruby 2.4 or later
+- Ruby 2.7 or later
 - git
 
 `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.

data/Rakefile

@@ -13,7 +13,7 @@ task test: [:base_test]
 namespace :build do
   desc 'Build gems for all platforms'
   task :all do
-    Bundler.with_clean_env do
+    Bundler.with_original_env do
       %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
         ENV['GEM_BUILD_FAKE_PLATFORM'] = name
         Rake::Task["build"].execute

data/fluentd.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 2.4'
+  gem.required_ruby_version = '>= 2.7'
 
   gem.add_runtime_dependency("bundler")
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
@@ -30,6 +30,11 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", ["~> 1.4"])
 
+  # gems that aren't default gems as of Ruby 3.4
+  gem.add_runtime_dependency("base64", ["~> 0.2"])
+  gem.add_runtime_dependency("csv", ["~> 3.2"])
+  gem.add_runtime_dependency("drb", ["~> 2.2"])
+
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
@@ -51,4 +56,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
   gem.add_development_dependency("async", "~> 1.23")
   gem.add_development_dependency("async-http", ">= 0.50.0")
+  gem.add_development_dependency("aws-sigv4", ["~> 1.8"])
+  gem.add_development_dependency("aws-sdk-core", ["~> 3.191"])
+  gem.add_development_dependency("rexml", ["~> 3.2"])
 end

data/lib/fluent/command/binlog_reader.rb

@@ -130,7 +130,7 @@ module BinlogReaderCommand
     private
 
     def configure_option_parser
-      @options.merge!(config_params: {})
+      @options[:config_params] = {}
 
       @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
 

data/lib/fluent/config/configure_proxy.rb

@@ -201,7 +201,7 @@ module Fluent
 
       def overwrite_defaults(other) # other is owner plugin's corresponding proxy
         self.defaults = self.defaults.merge(other.defaults)
-        self.sections.keys.each do |section_key|
+        self.sections.each_key do |section_key|
           if other.sections.has_key?(section_key)
             self.sections[section_key].overwrite_defaults(other.sections[section_key])
           end
@@ -274,7 +274,7 @@ module Fluent
         option_value_type!(name, opts, :deprecated, String)
         option_value_type!(name, opts, :obsoleted, String)
         if type == :enum
-          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
+          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?(Symbol)
             raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
           end
         end

data/lib/fluent/config/types.rb

@@ -123,7 +123,7 @@ module Fluent
 
       s = val.to_sym
       list = opts[:list]
-      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
+      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?(Symbol)
       unless list.include?(s)
         raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
       end

data/lib/fluent/configurable.rb

@@ -31,13 +31,13 @@ module Fluent
       super
       # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
       proxy = self.class.merged_configure_proxy
-      proxy.params.keys.each do |name|
+      proxy.params.each_key do |name|
         next if name.to_s.start_with?('@')
         if proxy.defaults.has_key?(name)
           instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
         end
       end
-      proxy.sections.keys.each do |name|
+      proxy.sections.each_key do |name|
         next if name.to_s.start_with?('@')
         subproxy = proxy.sections[name]
         if subproxy.multi?

data/lib/fluent/counter/mutex_hash.rb

@@ -54,7 +54,7 @@ module Fluent
               if mutex.try_lock
                 locks[key] = mutex
               else
-                locks.values.each(&:unlock)
+                locks.each_value(&:unlock)
                 locks = {}          # flush locked keys
                 break
               end

data/lib/fluent/fluent_log_event_router.rb

@@ -47,8 +47,6 @@ module Fluent
         # it's not suppressed in default event router for non-log-event events
         log_event_router.suppress_missing_match!
 
-        log_event_router = log_event_router
-
         unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
         unless unmatched_tags.empty?
           $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags

data/lib/fluent/plugin/buf_file.rb

@@ -56,7 +56,7 @@ module Fluent
 
         @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
 
-        multi_workers_configured = owner.system_config.workers > 1 ? true : false
+        multi_workers_configured = owner.system_config.workers > 1
 
         using_plugin_root_dir = false
         unless @path

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -399,7 +399,7 @@ module Fluent
           end
 
           if chunk.slice(0, 2) == BUFFER_HEADER
-            size = chunk.slice(2, 4).unpack('N').first
+            size = chunk.slice(2, 4).unpack1('N')
             if size
               return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
             end

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -238,17 +238,16 @@ module Fluent
           callback.call(file) if callback
         end
 
-        URI_PARSER = URI::Parser.new
         ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
 
         def encode_key(metadata)
           k = @key ? metadata.variables[@key] : metadata.tag
           k ||= ''
-          URI_PARSER.escape(k, ESCAPE_REGEXP)
+          URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
         end
 
         def decode_key(key)
-          URI_PARSER.unescape(key)
+          URI::DEFAULT_PARSER.unescape(key)
         end
 
         def create_new_chunk(path, metadata, perm)