fluentd 1.14.6 → 1.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c1188a4d037f788a1fbeec731c7117fef237cd4036cd5a5e5b891bba297d184
-  data.tar.gz: f3a1b383fd8d1ee4e5b431c92c83007b58c26f8d91bf1c9b0ce911662935c4e7
+  metadata.gz: 7bc368532203bcfd7b189e51e26a60f500db1306e6f5a4f94a03f333fb9af22d
+  data.tar.gz: b87ab437598cb9a41bdf6301e137baadcfc27c709d3e28354c2ba4e8ff8859ed
 SHA512:
-  metadata.gz: 9816373aac361da5cbd010e0720982f3f09ab59846e53ade46eccd225b7fc0201f8d02ddb009f3ffb813cfb41d92bd3360edbb7f5ea55ac644f46ad678521d21
-  data.tar.gz: d92e47b57424f825252d3fcc3204812d2747e634d98762a7843a6d68f1386258fc6302793311d21d82c3801e5f71f0fc30a4af1dc5464e2411a4d65b652b960b
+  metadata.gz: 45e167be3ee4b62e444b9f789694f7891590836b7e8888382a6368d3b49ddbece59f614b6eb20eb4b51f62b5e8fd4563e79304163fc78b3aaf4cc57d62a71546
+  data.tar.gz: 44112f3c79c48e09fb6c791a0edf6a465815d6ff3f3ccaa500cb4e6b7411c39a3ac7b587c11587b57568777786e3781e2ab68cc7e78088d4af386556b699e247

data/.github/workflows/linux-test.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['3.1', '3.0', '2.7', '2.6']
+        ruby-version: ['3.1', '3.0', '2.7']
         os: [ubuntu-latest]
         experimental: [false]
         include:

data/.github/workflows/windows-test.yaml

@@ -13,11 +13,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.7', '2.6']
+        ruby-version: ['3.1', '2.7']
         os:
           - windows-latest
         experimental: [false]
         include:
+          - ruby-version: head
+            os: windows-latest
+            experimental: true
           - ruby-version: '3.0.3'
             os: windows-latest
             experimental: false

data/CHANGELOG.md

@@ -1,3 +1,55 @@
+# v1.15
+
+## Release v1.15.0 - 2022/06/29
+
+### Enhancement
+
+* in_tail: Add log throttling in files based on group rules
+  https://github.com/fluent/fluentd/pull/3535
+  https://github.com/fluent/fluentd/pull/3771
+* Add `dump` command to fluent-ctl
+  https://github.com/fluent/fluentd/pull/3680
+* Handle YAML configuration format on configuration file
+  https://github.com/fluent/fluentd/pull/3712
+* Add `restart_worker_interval` parameter in `<system>` directive to set
+  interval to restart workers that has stopped for some reason.
+  https://github.com/fluent/fluentd/pull/3768
+
+### Bug fixes
+
+* out_forward: Fix to update timeout of cached sockets
+  https://github.com/fluent/fluentd/pull/3711
+* in_tail: Fix a possible crash on file rotation when `follow_inodes true`
+  https://github.com/fluent/fluentd/pull/3754
+* output: Fix a possible crash of flush thread
+  https://github.com/fluent/fluentd/pull/3755
+* in_tail: Fix crash bugs on Ruby 3.1 on Windows
+  https://github.com/fluent/fluentd/pull/3766
+* in_tail: Fix a bug that in_tail cannot open non-ascii path on Windows
+  https://github.com/fluent/fluentd/pull/3774
+* Fix a bug that fluentd doesn't release its own log file even after rotated by
+  external tools
+  https://github.com/fluent/fluentd/pull/3782
+
+### Misc
+
+* in_tail: Simplify TargetInfo related code
+  https://github.com/fluent/fluentd/pull/3489
+* Fix a wrong issue number in CHANGELOG
+  https://github.com/fluent/fluentd/pull/3700
+* server helper: Add comments to linger_timeout behavior about Windows
+  https://github.com/fluent/fluentd/pull/3701
+* service_discovery: Fix typo
+  https://github.com/fluent/fluentd/pull/3724
+* test: Fix unstable tests and warnings
+  https://github.com/fluent/fluentd/pull/3745
+  https://github.com/fluent/fluentd/pull/3753
+  https://github.com/fluent/fluentd/pull/3767
+  https://github.com/fluent/fluentd/pull/3783
+  https://github.com/fluent/fluentd/pull/3784
+  https://github.com/fluent/fluentd/pull/3785
+  https://github.com/fluent/fluentd/pull/3787
+
 # v1.14
 
 ## Release v1.14.6 - 2022/03/31
@@ -443,7 +495,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
 ### New feature
 
 * in_tail: Add `follow_inode` to support log rotation with wild card
-  https://github.com/fluent/fluentd/pull/2992
+  https://github.com/fluent/fluentd/pull/3182
 * in_tail: Handle linux capability
   https://github.com/fluent/fluentd/pull/3155
   https://github.com/fluent/fluentd/pull/3162

data/fluentd.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
-  gem.add_runtime_dependency("serverengine", [">= 2.2.5", "< 3.0.0"])
+  gem.add_runtime_dependency("serverengine", [">= 2.3.0", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
@@ -38,6 +38,7 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
+    gem.add_runtime_dependency("windows-api", ["~> 0.4.5"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
     gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
   end

data/lib/fluent/command/ctl.rb

@@ -36,6 +36,7 @@ module Fluent
         restart: "HUP",
         flush: "USR1",
         reload: "USR2",
+        dump: "CONT",
       }
       WINSVC_CONTROL_CODE_MAP = {
         shutdown: SERVICE_CONTROL_STOP,
@@ -44,6 +45,7 @@ module Fluent
         restart: 128,
         flush: 129,
         reload: SERVICE_CONTROL_PARAMCHANGE,
+        dump: 130,
       }
     else
       COMMAND_MAP = {
@@ -51,6 +53,7 @@ module Fluent
         restart: :HUP,
         flush: :USR1,
         reload: :USR2,
+        dump: :CONT,
       }
     end
 
@@ -146,7 +149,7 @@ module Fluent
         event = Win32::Event.open("#{prefix}#{suffix}")
         event.set
         event.close
-      rescue Errno::ENOENT => e
+      rescue Errno::ENOENT
         puts "Error: Cannot find the fluentd process with the event name: \"#{prefix}\""
       end
     end

data/lib/fluent/command/fluentd.rb

@@ -126,6 +126,16 @@ op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {
   opts[:without_source] = b
 }
 
+op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
+  if (s == 'yaml') || (s == 'yml')
+    opts[:config_file_type] = s.to_sym
+  elsif (s == 'guess')
+    opts[:config_file_type] = s.to_sym
+  else
+    usage '--config-file-type accepts yaml/yml or guess'
+  end
+}
+
 op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b|
   opts[:use_v1_config] = b
 }

data/lib/fluent/config/literal_parser.rb

@@ -191,9 +191,9 @@ worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
 EOM
         begin
           @eval_context.instance_eval(code)
-        rescue SetNil => e
+        rescue SetNil
           nil
-        rescue SetDefault => e
+        rescue SetDefault
           :default
         end
       end

data/lib/fluent/config/yaml_parser/fluent_value.rb

@@ -0,0 +1,47 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Config
+    module YamlParser
+      module FluentValue
+        JsonValue = Struct.new(:val) do
+          def to_s
+            val.to_json
+          end
+
+          def to_element
+            to_s
+          end
+        end
+
+        StringValue = Struct.new(:val, :context) do
+          def to_s
+            context.instance_eval("\"#{val}\"")
+          rescue Fluent::SetNil => _
+            ''
+          rescue Fluent::SetDefault => _
+            ':default'
+          end
+
+          def to_element
+            to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/loader.rb

@@ -0,0 +1,91 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'psych'
+require 'json'
+require 'fluent/config/error'
+require 'fluent/config/yaml_parser/fluent_value'
+
+# Based on https://github.com/eagletmt/hako/blob/34cdde06fe8f3aeafd794be830180c3cedfbb4dc/lib/hako/yaml_loader.rb
+
+module Fluent
+  module Config
+    module YamlParser
+      class Loader
+        INCLUDE_TAG = 'tag:include'.freeze
+        FLUENT_JSON_TAG = 'tag:fluent/json'.freeze
+        FLUENT_STR_TAG = 'tag:fluent/s'.freeze
+        SHOVEL = '<<'.freeze
+
+        def initialize(context = Kernel.binding)
+          @context = context
+          @current_path = nil
+        end
+
+        # @param [String] path
+        # @return [Hash]
+        def load(path)
+          class_loader = Psych::ClassLoader.new
+          scanner = Psych::ScalarScanner.new(class_loader)
+
+          visitor = Visitor.new(scanner, class_loader)
+
+          visitor._register_domain(INCLUDE_TAG) do |_, val|
+            load(path.parent.join(val))
+          end
+
+          visitor._register_domain(FLUENT_JSON_TAG) do |_, val|
+            Fluent::Config::YamlParser::FluentValue::JsonValue.new(val)
+          end
+
+          visitor._register_domain(FLUENT_STR_TAG) do |_, val|
+            Fluent::Config::YamlParser::FluentValue::StringValue.new(val, @context)
+          end
+
+          path.open do |f|
+            visitor.accept(Psych.parse(f))
+          end
+        end
+
+        class Visitor < Psych::Visitors::ToRuby
+          def initialize(scanner, class_loader)
+            super(scanner, class_loader)
+          end
+
+          def _register_domain(name, &block)
+            @domain_types.merge!({ name => [name, block] })
+          end
+
+          def revive_hash(hash, o)
+            super(hash, o).tap do |r|
+              if r[SHOVEL].is_a?(Hash)
+                h2 = {}
+                r.each do |k, v|
+                  if k == SHOVEL
+                    h2.merge!(v)
+                  else
+                    h2[k] = v
+                  end
+                end
+                r.replace(h2)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/parser.rb

@@ -0,0 +1,166 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/yaml_parser/section_builder'
+
+module Fluent
+  module Config
+    module YamlParser
+      class Parser
+        def initialize(config, indent: 2)
+          @base_indent = indent
+          @config = config
+        end
+
+        def build
+          s = @config['system'] && system_config_build(@config['system'])
+          c = @config['config'] && config_build(@config['config'], root: true)
+          RootBuilder.new(s, c)
+        end
+
+        private
+
+        def system_config_build(config)
+          section_build('system', config)
+        end
+
+        def config_build(config, indent: 0, root: false)
+          sb = SectionBodyBuilder.new(indent, root: root)
+          config.each do |c|
+            if (lc = c.delete('label'))
+              sb.add_section(label_build(lc, indent: indent))
+            end
+
+            if (sc = c.delete('source'))
+              sb.add_section(source_build(sc, indent: indent))
+            end
+
+            if (fc = c.delete('filter'))
+              sb.add_section(filter_build(fc, indent: indent))
+            end
+
+            if (mc = c.delete('match'))
+              sb.add_section(match_build(mc, indent: indent))
+            end
+
+            if (wc = c.delete('worker'))
+              sb.add_section(worker_build(wc, indent: indent))
+            end
+
+            included_sections_build(c, sb, indent: indent)
+          end
+
+          sb
+        end
+
+        def label_build(config, indent: 0)
+          config = config.dup
+          name = config.delete('$name')
+          c = config.delete('config')
+          SectionBuilder.new('label', config_build(c, indent: indent + @base_indent), indent, name)
+        end
+
+        def worker_build(config, indent: 0)
+          config = config.dup
+          num = config.delete('$arg')
+          c = config.delete('config')
+          SectionBuilder.new('worker', config_build(c, indent: indent + @base_indent), indent, num)
+        end
+
+        def source_build(config, indent: 0)
+          section_build('source', config, indent: indent)
+        end
+
+        def filter_build(config, indent: 0)
+          config = config.dup
+          tag = config.delete('$tag')
+          if tag.is_a?(Array)
+            section_build('filter', config, indent: indent, arg: tag&.join(','))
+          else
+            section_build('filter', config, indent: indent, arg: tag)
+          end
+        end
+
+        def match_build(config, indent: 0)
+          config = config.dup
+          tag = config.delete('$tag')
+          if tag.is_a?(Array)
+            section_build('match', config, indent: indent, arg: tag&.join(','))
+          else
+            section_build('match', config, indent: indent, arg: tag)
+          end
+        end
+
+        def included_sections_build(config, section_builder, indent: 0)
+          config.each_entry do |e|
+            k = e.keys.first
+            cc = e.delete(k)
+            case k
+            when 'label'
+              section_builder.add_section(label_build(cc, indent: indent))
+            when 'worker'
+              section_builder.add_section(worker_build(cc, indent: indent))
+            when 'source'
+              section_builder.add_section(source_build(cc, indent: indent))
+            when 'filter'
+              section_builder.add_section(filter_build(cc, indent: indent))
+            when 'match'
+              section_builder.add_section(match_build(cc, indent: indent))
+            end
+          end
+        end
+
+        def section_build(name, config, indent: 0, arg: nil)
+          sb = SectionBodyBuilder.new(indent + @base_indent)
+
+          if (v = config.delete('$type'))
+            sb.add_line('@type', v)
+          end
+
+          if (v = config.delete('$label'))
+            sb.add_line('@label', v)
+          end
+
+          if (v = config.delete('$id'))
+            sb.add_line('@id', v)
+          end
+
+          config.each do |key, val|
+            if val.is_a?(Array)
+              val.each do |v|
+                sb.add_section(section_build(key, v, indent: indent + @base_indent))
+              end
+            elsif val.is_a?(Hash)
+              harg = val.delete('$arg')
+              if harg.is_a?(Array)
+                # To prevent to generate invalid configuration for arg.
+                # "arg" should be String object and concatenated by ","
+                # when two or more objects are specified there.
+                sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg&.join(',')))
+              else
+                sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg))
+              end
+            else
+              sb.add_line(key, val)
+            end
+          end
+
+          SectionBuilder.new(name, sb, indent, arg)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/section_builder.rb

@@ -0,0 +1,107 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Config
+    module YamlParser
+      SectionBuilder = Struct.new(:name, :body, :indent_size, :arg) do
+        def to_s
+          indent = ' ' * indent_size
+
+          if arg && !arg.to_s.empty?
+            "#{indent}<#{name} #{arg}>\n#{body}\n#{indent}</#{name}>"
+          else
+            "#{indent}<#{name}>\n#{body}\n#{indent}</#{name}>"
+          end
+        end
+
+        def to_element
+          elem = body.to_element
+          elem.name = name
+          elem.arg = arg.to_s if arg
+          elem.v1_config = true
+          elem
+        end
+      end
+
+      class RootBuilder
+        def initialize(system, conf)
+          @system = system
+          @conf = conf
+        end
+
+        attr_reader :system, :conf
+
+        def to_element
+          Fluent::Config::Element.new('ROOT', '', {}, [@system, @conf].compact.map(&:to_element).flatten)
+        end
+
+        def to_s
+          s = StringIO.new(+'')
+          s.puts(@system.to_s) if @system
+          s.puts(@conf.to_s) if @conf
+
+          s.string
+        end
+      end
+
+      class SectionBodyBuilder
+        Row = Struct.new(:key, :value, :indent) do
+          def to_s
+            "#{indent}#{key} #{value}"
+          end
+        end
+
+        def initialize(indent, root: false)
+          @indent = ' ' * indent
+          @bodies = []
+          @root = root
+        end
+
+        def add_line(k, v)
+          @bodies << Row.new(k, v, @indent)
+        end
+
+        def add_section(section)
+          @bodies << section
+        end
+
+        def to_element
+          if @root
+            return @bodies.map(&:to_element)
+          end
+
+          not_section, section = @bodies.partition { |e| e.is_a?(Row) }
+          r = {}
+          not_section.each do |e|
+            v = e.value
+            r[e.key] = v.respond_to?(:to_element) ? v.to_element : v
+          end
+
+          if @root
+            section.map(&:to_element)
+          else
+            Fluent::Config::Element.new('', '', r, section.map(&:to_element))
+          end
+        end
+
+        def to_s
+          @bodies.map(&:to_s).join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser.rb

@@ -0,0 +1,56 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/yaml_parser/loader'
+require 'fluent/config/yaml_parser/parser'
+require 'pathname'
+
+module Fluent
+  module Config
+    module YamlParser
+      def self.parse(path)
+        context = Kernel.binding
+
+        unless context.respond_to?(:use_nil)
+          context.define_singleton_method(:use_nil) do
+            raise Fluent::SetNil
+          end
+        end
+
+        unless context.respond_to?(:use_default)
+          context.define_singleton_method(:use_default) do
+            raise Fluent::SetDefault
+          end
+        end
+
+        unless context.respond_to?(:hostname)
+          context.define_singleton_method(:hostname) do
+            Socket.gethostname
+          end
+        end
+
+        unless context.respond_to?(:worker_id)
+          context.define_singleton_method(:worker_id) do
+            ENV['SERVERENGINE_WORKER_ID'] || ''
+          end
+        end
+
+        s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))
+        Fluent::Config::YamlParser::Parser.new(s).build.to_element
+      end
+    end
+  end
+end

data/lib/fluent/config.rb

@@ -17,6 +17,7 @@
 require 'fluent/config/error'
 require 'fluent/config/element'
 require 'fluent/configurable'
+require 'fluent/config/yaml_parser'
 
 module Fluent
   module Config
@@ -25,7 +26,18 @@ module Fluent
     # @param additional_config [String] config which is added to last of config body
     # @param use_v1_config [Bool] config is formatted with v1 or not
     # @return [Fluent::Config]
-    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true)
+    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil)
+      if type == :guess
+        config_file_ext = File.extname(config_path)
+        if config_file_ext == '.yaml' || config_file_ext == '.yml'
+          type = :yaml
+        end
+      end
+
+      if type == :yaml || type == :yml
+        return Fluent::Config::YamlParser.parse(config_path)
+      end
+
       config_fname = File.basename(config_path)
       config_basedir = File.dirname(config_path)
       config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f|
@@ -36,6 +48,7 @@ module Fluent
         end
         s
       end
+
       Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config)
     end