fluentd 1.14.5 → 1.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98b1a3f0960184a4611f6c5139a0e4cb24cec72ec28581ad29d0012d32e1b9aa
-  data.tar.gz: 69cd5c67d4171fb9afb6a5c9d2c656b87921cc6e7a96aeed3017a62011010d63
+  metadata.gz: 767485d0b4b0b514e2bf2e26817b54b53e33d44fafd47ba8d86ce195ef274986
+  data.tar.gz: 525aa18805c114a792655dcd13ede2ed4c9370177d42efd8d33e8c9d1cce7d17
 SHA512:
-  metadata.gz: 32de1e2b6f891d041bac810aba2b1b9614b6cc72d2ffa22bb58cc4c9b61db3c54bac2d1bbfed56bfeae9147bacd840c37f1f43b8e841f15fcb7b07572301db83
-  data.tar.gz: 847ad4f2ff4615ee3fed3fe287d58a8734da7476a8ec8f444dcbf7ccec21b16a90e6c34351fc14737fa3ae0a9380e9cb53cbceda2873082cbaaa3e419b56c60a
+  metadata.gz: 7c6fe1fd468386c622ba02031e8dc24fe1d3d14666ae2bf195e3f34576cc0cbd50fa9e81519e747e101fcf2450cd51288b9e1b221df09dcb1fc7d7a5614b28ec
+  data.tar.gz: 7d17bb661cb80b6fcaf0348b0b5883ebc3a70b7c2a087091f586d7e83fe5b5805c18b785510a1689a81e17d20187126b53890e33c80ccc9addb26374befc5c0b

data/.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: Ask a Question
-    url: https://discuss.fluentd.org/
-    about: I have questions about Fluentd and plugins. Please ask and answer questions at https://discuss.fluentd.org/.
+    url: https://github.com/fluent/fluentd/discussions
+    about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions

data/.github/workflows/linux-test.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['3.0', '2.7', '2.6']
+        ruby-version: ['3.1', '3.0', '2.7']
         os: [ubuntu-latest]
         experimental: [false]
         include:

data/.github/workflows/macos-test.yaml

@@ -13,9 +13,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['head', '2.7']
+        ruby-version: ['3.1', '3.0', '2.7']
         os: [macos-latest]
         experimental: [true]
+        include:
+          - ruby-version: head
+            os: macos-latest
+            experimental: true
 
     name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:

data/.github/workflows/windows-test.yaml

@@ -13,14 +13,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.7', '2.6']
+        ruby-version: ['3.1', '2.7']
         os:
           - windows-latest
         experimental: [false]
         include:
-          - ruby-version: '3.0.2'
+          - ruby-version: head
             os: windows-latest
             experimental: true
+          - ruby-version: '3.0.3'
+            os: windows-latest
+            experimental: false
             # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
             # error code. In addition, we have to specify the path of fiddle by RUBYLIB
             # because RubyInstaller loads Ruby's bundled fiddle before initializing gem.
@@ -28,7 +31,7 @@ jobs:
             # * https://github.com/ruby/fiddle/issues/72
             # * https://bugs.ruby-lang.org/issues/17813
             # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
-            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.2/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
+            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib
 
     name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
@@ -37,9 +40,9 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby-version }}
-      - name: Add Fiddle 1.0.8
-        if: ${{ matrix.ruby-version == '3.0.2' }}
-        run: gem install fiddle --version 1.0.8
+      - name: Add Fiddle 1.1.0
+        if: ${{ matrix.ruby-version == '3.0.3' }}
+        run: gem install fiddle --version 1.1.0
       - name: Install dependencies
         run: ridk exec bundle install
       - name: Run tests

data/CHANGELOG.md

@@ -1,4 +1,104 @@
-# v1.14.5
+# v1.15
+
+## Release v1.15.1 - 2022/07/27
+
+### Bug Fix
+
+* Add support for concurrent append in out_file
+  https://github.com/fluent/fluentd/pull/3808
+
+### Misc
+
+* in_tail: Show more information on skipping update_watcher
+  https://github.com/fluent/fluentd/pull/3829
+
+## Release v1.15.0 - 2022/06/29
+
+### Enhancement
+
+* in_tail: Add log throttling in files based on group rules
+  https://github.com/fluent/fluentd/pull/3535
+  https://github.com/fluent/fluentd/pull/3771
+* Add `dump` command to fluent-ctl
+  https://github.com/fluent/fluentd/pull/3680
+* Handle YAML configuration format on configuration file
+  https://github.com/fluent/fluentd/pull/3712
+* Add `restart_worker_interval` parameter in `<system>` directive to set
+  interval to restart workers that has stopped for some reason.
+  https://github.com/fluent/fluentd/pull/3768
+
+### Bug fixes
+
+* out_forward: Fix to update timeout of cached sockets
+  https://github.com/fluent/fluentd/pull/3711
+* in_tail: Fix a possible crash on file rotation when `follow_inodes true`
+  https://github.com/fluent/fluentd/pull/3754
+* output: Fix a possible crash of flush thread
+  https://github.com/fluent/fluentd/pull/3755
+* in_tail: Fix crash bugs on Ruby 3.1 on Windows
+  https://github.com/fluent/fluentd/pull/3766
+* in_tail: Fix a bug that in_tail cannot open non-ascii path on Windows
+  https://github.com/fluent/fluentd/pull/3774
+* Fix a bug that fluentd doesn't release its own log file even after rotated by
+  external tools
+  https://github.com/fluent/fluentd/pull/3782
+
+### Misc
+
+* in_tail: Simplify TargetInfo related code
+  https://github.com/fluent/fluentd/pull/3489
+* Fix a wrong issue number in CHANGELOG
+  https://github.com/fluent/fluentd/pull/3700
+* server helper: Add comments to linger_timeout behavior about Windows
+  https://github.com/fluent/fluentd/pull/3701
+* service_discovery: Fix typo
+  https://github.com/fluent/fluentd/pull/3724
+* test: Fix unstable tests and warnings
+  https://github.com/fluent/fluentd/pull/3745
+  https://github.com/fluent/fluentd/pull/3753
+  https://github.com/fluent/fluentd/pull/3767
+  https://github.com/fluent/fluentd/pull/3783
+  https://github.com/fluent/fluentd/pull/3784
+  https://github.com/fluent/fluentd/pull/3785
+  https://github.com/fluent/fluentd/pull/3787
+
+# v1.14
+
+## Release v1.14.6 - 2022/03/31
+
+### Enhancement
+
+* Enable server plugins to specify socket-option `SO_LINGER`
+  https://github.com/fluent/fluentd/pull/3644
+* Add `--umask` command line parameter
+  https://github.com/fluent/fluentd/pull/3671
+  https://github.com/fluent/fluentd/pull/3679
+
+### Bug fixes
+
+* Fix metric name typo
+  https://github.com/fluent/fluentd/pull/3630
+  https://github.com/fluent/fluentd/pull/3673
+* Apply modifications in pipeline to the records being passed to `@ERROR` label
+  https://github.com/fluent/fluentd/pull/3631
+* Fix wrong calculation of retry interval
+  https://github.com/fluent/fluentd/pull/3640
+  https://github.com/fluent/fluentd/pull/3649
+  https://github.com/fluent/fluentd/pull/3685
+  https://github.com/fluent/fluentd/pull/3686
+* Support IPv6 address for `rpc_endpoint` in `system` config
+  https://github.com/fluent/fluentd/pull/3641
+
+### Misc
+
+* CI: Support Ruby 3.1 except Windows
+  https://github.com/fluent/fluentd/pull/3619
+* Switch to GitHub Discussions
+  https://github.com/fluent/fluentd/pull/3654
+* Fix CHANGELOG.md heading styles
+  https://github.com/fluent/fluentd/pull/3648
+* Declare `null_value_pattern` as `regexp`
+  https://github.com/fluent/fluentd/pull/3650
 
 ## Release v1.14.5 - 2022/02/09
 
@@ -22,8 +122,6 @@
 * Fix "ArgumentError: unknown keyword: :logger" on Windows with Ruby 3.1
   https://github.com/fluent/fluentd/pull/3592
 
-# v1.14.4
-
 ## Release v1.14.4 - 2022/01/06
 
 ### Enhancement
@@ -41,8 +139,6 @@
   https://github.com/fluent/fluentd/pull/3574
   https://github.com/fluent/fluentd/pull/3577
 
-# v1.14.3
-
 ## Release v1.14.3 - 2021/11/26
 
 ### Enhancement
@@ -71,8 +167,6 @@
   newer version is required to implement additional `fluent-ctl` commands.
   https://github.com/fluent/fluentd/pull/3556
 
-# v1.14.2
-
 ## Release v1.14.2 - 2021/10/29
 
 IMPORTANT: This release contain the fix for CVE-2021-41186 -
@@ -84,6 +178,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
 ### Enhancement
 
 * fluent-cat: Add `--event-time` option to send specified event time for testing.
+  https://github.com/fluent/fluentd/pull/3528
 
 ### Bug fix
 
@@ -92,8 +187,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
 * Fixed ReDoS vulnerability in parser_apache2.
   This vulnerability is caused by a certain pattern of a broken apache log.
 
-# v1.14.1
-
 ## Release v1.14.1 - 2021/09/29
 
 ### Enhancement
@@ -123,8 +216,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
   next_retry_seconds -> next_retry_time)
   https://github.com/fluent/fluentd/pull/3518
 
-# v1.14.0
-
 ## Release v1.14.0 - 2021/08/30
 
 ### Enhancement
@@ -167,7 +258,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
   https://github.com/fluent/fluentd/pull/3479
   https://github.com/fluent/fluentd/pull/3484
 
-# v1.13.3
+# v1.13
 
 ## Release v1.13.3 - 2021/07/27
 
@@ -189,10 +280,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
 * Remove needless spaces in a sample config file
   https://github.com/fluent/fluentd/pull/3456
 
-### Enhancement
-
-# v1.13.2
-
 ## Release v1.13.2 - 2021/07/12
 
 ### Enhancement
@@ -228,8 +315,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
   by security scanning tools.
   https://github.com/fluent/fluentd/pull/3450
 
-# v1.13.1
-
 ## Release v1.13.1 - 2021/06/25
 
 ### Bug fix
@@ -254,8 +339,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
 * CI: Dropped to run CI for Ruby 2.5
   https://github.com/fluent/fluentd/pull/3412
 
-# v1.13
-
 ## Release v1.13.0 - 2021/05/29
 
 ### Enhancement
@@ -303,6 +386,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
 ### Bug fix
 
 * in_tail: Fix a bug that refresh_watcher fails to handle file rotations
+  https://github.com/fluent/fluentd/pull/3393
 
 ## Release v1.12.3 - 2021/04/23
 
@@ -423,7 +507,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
 ### New feature
 
 * in_tail: Add `follow_inode` to support log rotation with wild card
-  https://github.com/fluent/fluentd/pull/2992
+  https://github.com/fluent/fluentd/pull/3182
 * in_tail: Handle linux capability
   https://github.com/fluent/fluentd/pull/3155
   https://github.com/fluent/fluentd/pull/3162

data/CONTRIBUTING.md

@@ -6,7 +6,7 @@ We'd love your contribution. Here are the guidelines!
 
 RESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you.
 
-If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://discuss.fluentd.org/).
+If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://github.com/fluent/fluentd/discussions).
 Don't use Github issue for asking questions. Here are examples:
 
 - I installed xxx plugin but it doesn't work. Why?

data/MAINTAINERS.md

@@ -2,10 +2,10 @@
 
 - [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
 - [Okkez](https://github.com/okkez)
-- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://www.calyptia.com/)
+- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://calyptia.com/)
 - [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
 - [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
-- [Eduardo Silva](https://github.com/edsiper), [Arm Treasure Data](https://www.treasuredata.com/)
+- [Eduardo Silva](https://github.com/edsiper), [Calyptia](https://calyptia/)
 - [Fujimoto Seiji](https://github.com/fujimots), [ClearCode](https://www.clear-code.com/)
 - [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)
 - [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)

data/README.md

@@ -56,7 +56,7 @@ You can run specified test via `TEST` environment variable:
 - Website: https://www.fluentd.org/
 - Documentation: https://docs.fluentd.org/
 - Project repository: https://github.com/fluent
-- Discussion: https://discuss.fluentd.org/
+- Discussion: https://github.com/fluent/fluentd/discussions
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter
 - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)

data/fluentd.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
-  gem.add_runtime_dependency("serverengine", [">= 2.2.5", "< 3.0.0"])
+  gem.add_runtime_dependency("serverengine", [">= 2.3.0", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
@@ -38,6 +38,7 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
+    gem.add_runtime_dependency("windows-api", ["~> 0.4.5"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
     gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
   end

data/lib/fluent/command/ctl.rb

@@ -36,6 +36,7 @@ module Fluent
         restart: "HUP",
         flush: "USR1",
         reload: "USR2",
+        dump: "CONT",
       }
       WINSVC_CONTROL_CODE_MAP = {
         shutdown: SERVICE_CONTROL_STOP,
@@ -44,6 +45,7 @@ module Fluent
         restart: 128,
         flush: 129,
         reload: SERVICE_CONTROL_PARAMCHANGE,
+        dump: 130,
       }
     else
       COMMAND_MAP = {
@@ -51,6 +53,7 @@ module Fluent
         restart: :HUP,
         flush: :USR1,
         reload: :USR2,
+        dump: :CONT,
       }
     end
 
@@ -146,7 +149,7 @@ module Fluent
         event = Win32::Event.open("#{prefix}#{suffix}")
         event.set
         event.close
-      rescue Errno::ENOENT => e
+      rescue Errno::ENOENT
         puts "Error: Cannot find the fluentd process with the event name: \"#{prefix}\""
       end
     end

data/lib/fluent/command/fluentd.rb

@@ -81,6 +81,10 @@ op.on('--group GROUP', "change group") {|s|
   opts[:chgroup] = s
 }
 
+op.on('--umask UMASK', "change umask") {|s|
+  opts[:chumask] = s
+}
+
 op.on('-o', '--log PATH', "log file path") {|s|
   opts[:log_path] = s
 }
@@ -122,6 +126,16 @@ op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {
   opts[:without_source] = b
 }
 
+op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
+  if (s == 'yaml') || (s == 'yml')
+    opts[:config_file_type] = s.to_sym
+  elsif (s == 'guess')
+    opts[:config_file_type] = s.to_sym
+  else
+    usage '--config-file-type accepts yaml/yml or guess'
+  end
+}
+
 op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b|
   opts[:use_v1_config] = b
 }

data/lib/fluent/config/literal_parser.rb

@@ -191,9 +191,9 @@ worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
 EOM
         begin
           @eval_context.instance_eval(code)
-        rescue SetNil => e
+        rescue SetNil
           nil
-        rescue SetDefault => e
+        rescue SetDefault
           :default
         end
       end

data/lib/fluent/config/yaml_parser/fluent_value.rb

@@ -0,0 +1,47 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Config
+    module YamlParser
+      module FluentValue
+        JsonValue = Struct.new(:val) do
+          def to_s
+            val.to_json
+          end
+
+          def to_element
+            to_s
+          end
+        end
+
+        StringValue = Struct.new(:val, :context) do
+          def to_s
+            context.instance_eval("\"#{val}\"")
+          rescue Fluent::SetNil => _
+            ''
+          rescue Fluent::SetDefault => _
+            ':default'
+          end
+
+          def to_element
+            to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/loader.rb

@@ -0,0 +1,91 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'psych'
+require 'json'
+require 'fluent/config/error'
+require 'fluent/config/yaml_parser/fluent_value'
+
+# Based on https://github.com/eagletmt/hako/blob/34cdde06fe8f3aeafd794be830180c3cedfbb4dc/lib/hako/yaml_loader.rb
+
+module Fluent
+  module Config
+    module YamlParser
+      class Loader
+        INCLUDE_TAG = 'tag:include'.freeze
+        FLUENT_JSON_TAG = 'tag:fluent/json'.freeze
+        FLUENT_STR_TAG = 'tag:fluent/s'.freeze
+        SHOVEL = '<<'.freeze
+
+        def initialize(context = Kernel.binding)
+          @context = context
+          @current_path = nil
+        end
+
+        # @param [String] path
+        # @return [Hash]
+        def load(path)
+          class_loader = Psych::ClassLoader.new
+          scanner = Psych::ScalarScanner.new(class_loader)
+
+          visitor = Visitor.new(scanner, class_loader)
+
+          visitor._register_domain(INCLUDE_TAG) do |_, val|
+            load(path.parent.join(val))
+          end
+
+          visitor._register_domain(FLUENT_JSON_TAG) do |_, val|
+            Fluent::Config::YamlParser::FluentValue::JsonValue.new(val)
+          end
+
+          visitor._register_domain(FLUENT_STR_TAG) do |_, val|
+            Fluent::Config::YamlParser::FluentValue::StringValue.new(val, @context)
+          end
+
+          path.open do |f|
+            visitor.accept(Psych.parse(f))
+          end
+        end
+
+        class Visitor < Psych::Visitors::ToRuby
+          def initialize(scanner, class_loader)
+            super(scanner, class_loader)
+          end
+
+          def _register_domain(name, &block)
+            @domain_types.merge!({ name => [name, block] })
+          end
+
+          def revive_hash(hash, o)
+            super(hash, o).tap do |r|
+              if r[SHOVEL].is_a?(Hash)
+                h2 = {}
+                r.each do |k, v|
+                  if k == SHOVEL
+                    h2.merge!(v)
+                  else
+                    h2[k] = v
+                  end
+                end
+                r.replace(h2)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end