fluentd 1.13.3 → 1.16.5

data/lib/fluent/plugin.rb

@@ -36,8 +36,9 @@ module Fluent
     FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
     STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
     SD_REGISTRY        = Registry.new(:sd,        'fluent/plugin/sd_',         dir_search_prefix: 'sd_')
+    METRICS_REGISTRY   = Registry.new(:metrics,   'fluent/plugin/metrics_',    dir_search_prefix: 'metrics_')
 
-    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]
 
     def self.register_input(type, klass)
       register_impl('input', INPUT_REGISTRY, type, klass)
@@ -59,6 +60,10 @@ module Fluent
       register_impl('sd', SD_REGISTRY, type, klass)
     end
 
+    def self.register_metrics(type, klass)
+      register_impl('metrics', METRICS_REGISTRY, type, klass)
+    end
+
     def self.register_parser(type, klass_or_proc)
       if klass_or_proc.is_a?(Regexp)
         # This usage is not recommended for new API
@@ -121,6 +126,10 @@ module Fluent
       new_impl('sd', SD_REGISTRY, type, parent)
     end
 
+    def self.new_metrics(type, parent: nil)
+      new_impl('metrics', METRICS_REGISTRY, type, parent)
+    end
+
     class << self
       # This should be defined for fluent-plugin-config-formatter type arguments.
       alias_method :new_service_discovery, :new_sd

data/lib/fluent/plugin_helper/child_process.rb

@@ -346,6 +346,9 @@ module Fluent
           if cb
             cb.call(process_info.exit_status) rescue nil
           end
+          process_info.readio&.close rescue nil
+          process_info.writeio&.close rescue nil
+          process_info.stderrio&.close rescue nil
         end
         thread[:_fluentd_plugin_helper_child_process_running] = true
         thread[:_fluentd_plugin_helper_child_process_pid] = pid

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -29,6 +29,9 @@ module Fluent
         if @_event_emitter_lazy_init
           @router = @primary_instance.router
         end
+        if @router.respond_to?(:caller_plugin_id=)
+          @router.caller_plugin_id = self.plugin_id
+        end
         @router
       end
 
@@ -47,7 +50,11 @@ module Fluent
 
       def event_emitter_router(label_name)
         if label_name
-          Engine.root_agent.find_label(label_name).event_router
+          if label_name == "@ROOT"
+            Engine.root_agent.event_router
+          else
+            Engine.root_agent.find_label(label_name).event_router
+          end
         elsif self.respond_to?(:as_secondary) && self.as_secondary
           if @primary_instance.has_router?
             @_event_emitter_lazy_init = true

data/lib/fluent/plugin_helper/event_loop.rb

@@ -99,7 +99,7 @@ module Fluent
 
       def shutdown
         @_event_loop_mutex.synchronize do
-          @_event_loop_attached_watchers.reverse.each do |w|
+          @_event_loop_attached_watchers.reverse_each do |w|
             if w.attached?
               begin
                 w.detach
@@ -116,7 +116,7 @@ module Fluent
       def after_shutdown
         timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
         @_event_loop_mutex.synchronize do
-          @_event_loop.watchers.reverse.each do |w|
+          @_event_loop.watchers.reverse_each do |w|
             begin
               w.detach
             rescue => e

data/lib/fluent/plugin_helper/http_server/server.rb

@@ -21,6 +21,7 @@ require 'async/http/endpoint'
 require 'fluent/plugin_helper/http_server/app'
 require 'fluent/plugin_helper/http_server/router'
 require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/log/console_adapter'
 
 module Fluent
   module PluginHelper
@@ -38,7 +39,7 @@ module Fluent
           scheme = tls_context ? 'https' : 'http'
           @uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
           @router = Router.new(default_app)
-          @reactor = Async::Reactor.new(nil, logger: @logger)
+          @reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(@logger))
 
           opts = if tls_context
                    { ssl_context: tls_context }

data/lib/fluent/plugin_helper/metrics.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'forwardable'
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin_helper/timer'
+require 'fluent/config/element'
+require 'fluent/configurable'
+require 'fluent/system_config'
+
+module Fluent
+  module PluginHelper
+    module Metrics
+      include Fluent::SystemConfig::Mixin
+
+      attr_reader :_metrics # For tests.
+
+      def initialize
+        super
+        @_metrics_started = false
+        @_metrics = {} # usage => metrics_state
+      end
+
+      def configure(conf)
+        super
+
+        @plugin_type_or_id = if self.plugin_id_configured?
+                               self.plugin_id
+                             else
+                               if type = (conf["@type"] || conf["type"])
+                                 "#{type}.#{self.plugin_id}"
+                               else
+                                 "#{self.class.to_s.split("::").last.downcase}.#{self.plugin_id}"
+                               end
+                             end
+      end
+
+      def metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)
+        metrics = if system_config.metrics
+                    Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)
+                  else
+                    Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)
+                  end
+        config = if system_config.metrics
+                   system_config.metrics.corresponding_config_element
+                 else
+                   Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])
+                 end
+        metrics.use_gauge_metric = prefer_gauge
+        metrics.configure(config)
+        # For multi workers environment, cmetrics should be distinguish with static labels.
+        if Fluent::Engine.system_config.workers > 1
+          labels.merge!(worker_id: fluentd_worker_id.to_s)
+        end
+        labels.merge!(plugin: @plugin_type_or_id)
+        metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
+
+        @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
+
+        metrics
+      end
+
+      def metrics_operate(method_name, &block)
+        @_metrics.each_pair do |key, m|
+          begin
+            block.call(s) if block_given?
+            m.__send__(method_name)
+          rescue => e
+            log.error "unexpected error while #{method_name}", key: key, metrics: m, error: e
+          end
+        end
+      end
+
+      def start
+        super
+
+        metrics_operate(:start)
+        @_metrics_started = true
+      end
+
+      def stop
+        super
+        # timer stops automatically in super
+        metrics_operate(:stop)
+      end
+
+      def before_shutdown
+        metrics_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        metrics_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        metrics_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        metrics_operate(:close)
+        super
+      end
+
+      def terminate
+        metrics_operate(:terminate)
+        @_metrics = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -119,7 +119,7 @@ module Fluent
         def self.validate_dot_keys(keys)
           keys.each { |key|
             next unless key.is_a?(String)
-            if /\s+/.match(key)
+            if /\s+/.match?(key)
               raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
             end
           }

data/lib/fluent/plugin_helper/retry_state.rb

@@ -44,6 +44,8 @@ module Fluent
 
           @timeout = timeout
           @timeout_at = @start + timeout
+          @has_reached_timeout = false
+          @has_timed_out = false
           @current = :primary
 
           if randomize_width < 0 || randomize_width > 0.5
@@ -98,7 +100,7 @@ module Fluent
               naive
             end
           elsif @current == :secondary
-            naive = naive_next_time(@steps - @secondary_transition_steps + 1)
+            naive = naive_next_time(@steps - @secondary_transition_steps)
             if naive >= @timeout_at
               @timeout_at
             else
@@ -123,7 +125,15 @@ module Fluent
             @current = :secondary
             @secondary_transition_steps = @steps
           end
+
           @next_time = calc_next_time
+
+          if @has_reached_timeout
+            @has_timed_out = @next_time >= @timeout_at
+          else
+            @has_reached_timeout = @next_time >= @timeout_at
+          end
+
           nil
         end
 
@@ -135,7 +145,7 @@ module Fluent
           if @forever
             false
           else
-            @next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps)
+            @has_timed_out || !!(@max_steps && @steps >= @max_steps)
           end
         end
       end
@@ -165,7 +175,7 @@ module Fluent
         end
 
         def calc_interval(num)
-          interval = raw_interval(num - 1)
+          interval = raw_interval(num)
           if @max_interval && interval > @max_interval
             @max_interval
           else
@@ -175,7 +185,7 @@ module Fluent
               # Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.
               until interval.finite?
                 num -= 1
-                interval = raw_interval(num - 1)
+                interval = raw_interval(num)
               end
               interval
             end

data/lib/fluent/plugin_helper/server.rb

@@ -80,8 +80,8 @@ module Fluent
         raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
         raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
 
-        if proto == :tcp || proto == :tls # default linger_timeout only for server
-          socket_options[:linger_timeout] ||= 0
+        if proto == :tcp || proto == :tls
+          socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
         end
 
         socket_option_validate!(proto, **socket_options)
@@ -132,8 +132,8 @@ module Fluent
         raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
         raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
 
-        if proto == :tcp || proto == :tls # default linger_timeout only for server
-          socket_options[:linger_timeout] ||= 0
+        if proto == :tcp || proto == :tls
+          socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
         end
 
         unless socket
@@ -263,6 +263,25 @@ module Fluent
         include Fluent::Configurable
         config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
           config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
+
+          ### Socket Params ###
+
+          # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
+          # Set positive value if needing to send FIN on closing on non-Windows.
+          # (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
+          # See `socket_option.rb`.)
+          # NOTE:
+            # Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.
+            # This is because there is no positive reason to do so.
+            # `linger_timeout` option in particular needs to be defined here
+            # although it can be specified from each plugin as well.
+            # This is because this helper fixes the default value to `0` for its own reason
+            # and it has a critical effect on the behavior.
+          desc 'The timeout time used to set linger option.'
+          config_param :linger_timeout, :integer, default: 0
+
+          ### TLS Params ###
+
           config_param :version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
           config_param :min_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
           config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
@@ -526,6 +545,10 @@ module Fluent
               data = @sock.recv(@max_bytes, @flags)
             rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
+            rescue Errno::EMSGSIZE
+              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
+              @log.info "A received data was ignored since it was too large."
+              return
             end
             @callback.call(data)
           rescue => e
@@ -539,6 +562,10 @@ module Fluent
               data, addr = @sock.recvfrom(@max_bytes)
             rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
+            rescue Errno::EMSGSIZE
+              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
+              @log.info "A received data was ignored since it was too large."
+              return
             end
             @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
           rescue => e
@@ -709,13 +736,15 @@ module Fluent
                 return true
               end
             rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
+              peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
               @log.trace "unexpected error before accepting TLS connection",
-                         host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
+                         addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
               close rescue nil
             rescue OpenSSL::SSL::SSLError => e
+              peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
               # Use same log level as on_readable
               @log.warn "unexpected error before accepting TLS connection by OpenSSL",
-                        host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
+                        addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
               close rescue nil
             end
 

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -44,7 +44,7 @@ module Fluent
 
         @discovery_manager.start
         unless @discovery_manager.static_config?
-          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_iterval) do
+          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_interval) do
             @discovery_manager.run_once
           end
         end
@@ -96,7 +96,7 @@ module Fluent
       # @param custom_build_method [Proc]
       def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
         @_plugin_helper_service_discovery_title = title
-        @_plugin_helper_service_discovery_iterval = interval
+        @_plugin_helper_service_discovery_interval = interval
 
         @discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
           log: log,

data/lib/fluent/plugin_helper/socket.rb

@@ -96,6 +96,7 @@ module Fluent
           enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
           cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
           cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
+          connect_timeout: nil,
           **kwargs, &block)
 
         host_is_ipaddress = IPAddr.new(host) rescue false
@@ -158,13 +159,23 @@ module Fluent
         end
         Fluent::TLS.set_version_to_context(context, version, min_version, max_version)
 
-        tcpsock = socket_create_tcp(host, port, **kwargs)
+        tcpsock = socket_create_tcp(host, port, connect_timeout: connect_timeout, **kwargs)
         sock = WrappedSocket::TLS.new(tcpsock, context)
         sock.sync_close = true
         sock.hostname = fqdn if verify_fqdn && fqdn && sock.respond_to?(:hostname=)
 
         log.trace "entering TLS handshake"
-        sock.connect
+        if connect_timeout
+          begin
+            Timeout.timeout(connect_timeout) { sock.connect }
+          rescue Timeout::Error
+            log.warn "timeout while connecting tls session", host: host
+            sock.close rescue nil
+            raise
+          end
+        else
+          sock.connect
+        end
 
         begin
           if verify_fqdn

data/lib/fluent/plugin_helper/thread.rb

@@ -101,16 +101,16 @@ module Fluent
       end
 
       def thread_exist?(title)
-        @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
+        @_threads.values.count{|thread| title == thread[:_fluentd_plugin_helper_thread_title] } > 0
       end
 
       def thread_started?(title)
-        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
         t && t[:_fluentd_plugin_helper_thread_started]
       end
 
       def thread_running?(title)
-        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
         t && t[:_fluentd_plugin_helper_thread_running]
       end
 

data/lib/fluent/plugin_helper.rb

@@ -32,6 +32,7 @@ require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/record_accessor'
 require 'fluent/plugin_helper/compat_parameters'
 require 'fluent/plugin_helper/service_discovery'
+require 'fluent/plugin_helper/metrics'
 
 module Fluent
   module PluginHelper

data/lib/fluent/plugin_id.rb

@@ -15,6 +15,7 @@
 #
 
 require 'set'
+require 'fluent/env'
 require 'fluent/variable_store'
 
 module Fluent
@@ -48,7 +49,7 @@ module Fluent
         # Thread::Backtrace::Location#path returns base filename or absolute path.
         # #absolute_path returns absolute_path always.
         # https://bugs.ruby-lang.org/issues/12159
-        if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+        if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
           return true
         end
       end
@@ -76,7 +77,7 @@ module Fluent
 
       # Fluent::Plugin::Base#fluentd_worker_id
       dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
-      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(dir)
+      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
       @_plugin_root_dir = dir.freeze
       dir
     end

data/lib/fluent/registry.rb

@@ -45,7 +45,8 @@ module Fluent
       if value = @map[type]
         return value
       end
-      raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"  # TODO error class
+      raise NotFoundPluginError.new("Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins",
+                                    kind: @kind, type: type)
     end
 
     def reverse_lookup(value)

data/lib/fluent/root_agent.rb

@@ -55,9 +55,11 @@ module Fluent
       @suppress_emit_error_log_interval = 0
       @next_emit_error_log_time = nil
       @without_source = false
+      @enable_input_metrics = false
 
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
+      @enable_input_metrics = !!system_config.enable_input_metrics
     end
 
     attr_reader :inputs
@@ -131,6 +133,7 @@ module Fluent
         end
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
+        raise ConfigError, "@ROOT for <label> is not permitted, reserved for getting root router" if name == '@ROOT'
 
         if name == ERROR_LABEL
           error_label_config = e
@@ -315,6 +318,9 @@ module Fluent
       # See also 'fluentd/plugin/input.rb'
       input.context_router = @event_router
       input.configure(conf)
+      if @enable_input_metrics
+        @event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
+      end
       @inputs << input
 
       input

data/lib/fluent/rpc.rb

@@ -20,9 +20,10 @@ module Fluent
   module RPC
     class Server
       def initialize(endpoint, log)
-        bind, port = endpoint.split(':')
-        @bind = bind
-        @port = port
+        m = endpoint.match(/^\[?(?<host>[0-9a-zA-Z:\-\.]+)\]?:(?<port>[0-9]+)$/)
+        raise Fluent::ConfigError, "Invalid rpc_endpoint: #{endpoint}" unless m
+        @bind = m[:host]
+        @port = m[:port]
         @log = log
 
         @server = WEBrick::HTTPServer.new(