fluentd 1.13.3 → 1.16.5

data/lib/fluent/plugin/in_tcp.rb

@@ -36,10 +36,16 @@ module Fluent::Plugin
     desc "The field name of the client's address."
     config_param :source_address_key, :string, default: nil
 
+    # Setting default to nil for backward compatibility
+    desc "The max bytes of message."
+    config_param :message_length_limit, :size, default: nil
+
     config_param :blocking_timeout, :time, default: 0.5
 
     desc 'The payload is read up to this character.'
     config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
+    config_param :send_keepalive_packet, :bool, default: false
 
     # in_forward like host/network restriction
     config_section :security, required: false, multi: false do
@@ -100,8 +106,9 @@ module Fluent::Plugin
 
       log.info "listening tcp socket", bind: @bind, port: @port
       del_size = @delimiter.length
+      discard_till_next_delimiter = false
       if @_extract_enabled && @_extract_tag_key
-        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
             conn.close
             next
@@ -114,6 +121,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -129,9 +146,18 @@ module Fluent::Plugin
             end
           end
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       else
-        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
             conn.close
             next
@@ -145,6 +171,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -159,6 +195,15 @@ module Fluent::Plugin
           end
           router.emit_stream(@tag, es)
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       end
     end

data/lib/fluent/plugin/input.rb

@@ -27,7 +27,45 @@ module Fluent
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      helpers_internal :event_emitter
+      helpers_internal :event_emitter, :metrics
+
+      def initialize
+        super
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @counter_mutex = Mutex.new
+        @enable_size_metrics = false
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def configure(conf)
+        super
+
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_records", help_text: "Number of count emit records")
+        @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'input' => stats }
+      end
+
+      def metric_callback(es)
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
+      end
 
       def multi_workers_ready?
         false

data/lib/fluent/plugin/metrics.rb

@@ -0,0 +1,119 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/unique_id'
+require 'fluent/plugin_id'
+
+module Fluent
+  module Plugin
+    class Metrics < Base
+      include PluginId
+      include PluginLoggerMixin
+      include UniqueId::Mixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :metrics
+
+      config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
+      config_param :labels, :hash, default: {}
+
+      attr_reader :use_gauge_metric
+      attr_reader :has_methods_for_gauge, :has_methods_for_counter
+
+      def initialize
+        super
+
+        @has_methods_for_counter = false
+        @has_methods_for_gauge = false
+        @use_gauge_metric = false
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          @has_methods_for_gauge = has_methods_for_gauge?
+        else
+          @has_methods_for_counter = has_methods_for_counter?
+        end
+      end
+
+      # Some metrics should be counted by gauge.
+      # ref: https://prometheus.io/docs/concepts/metric_types/#gauge
+      def use_gauge_metric=(use_gauge_metric=false)
+        @use_gauge_metric = use_gauge_metric
+      end
+
+      def create(namespace:, subsystem:,name:,help_text:,labels: {})
+        # This API is for cmetrics type.
+      end
+
+      def get
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def inc
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def dec
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def add(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def sub(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def set(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      private
+
+      def has_methods_for_counter?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
+        end
+      end
+
+      def has_methods_for_gauge?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/metrics_local.rb

@@ -0,0 +1,96 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+
+module Fluent
+  module Plugin
+    class LocalMetrics < Metrics
+      Fluent::Plugin.register_metrics('local', self)
+
+      def initialize
+        super
+        @store = 0
+        @monitor = Monitor.new
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          class << self
+            alias_method :dec, :dec_gauge
+            alias_method :set, :set_gauge
+            alias_method :sub, :sub_gauge
+          end
+        else
+          class << self
+            alias_method :set, :set_counter
+          end
+        end
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def get
+        @monitor.synchronize do
+          @store
+        end
+      end
+
+      def inc
+        @monitor.synchronize do
+          @store += 1
+        end
+      end
+
+      def dec_gauge
+        @monitor.synchronize do
+          @store -= 1
+        end
+      end
+
+      def add(value)
+        @monitor.synchronize do
+          @store += value
+        end
+      end
+
+      def sub_gauge(value)
+        @monitor.synchronize do
+          @store -= value
+        end
+      end
+
+      def set_counter(value)
+        return if @store > value
+
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+
+      def set_gauge(value)
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -27,6 +27,7 @@ module Fluent
       include PluginHelper::Mixin # for event_emitter
 
       helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+      helpers_internal :metrics
 
       config_section :store, param_name: :stores, multi: true, required: true do
         config_argument :arg, :string, default: ''
@@ -46,11 +47,40 @@ module Fluent
 
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
         # @write_count = 0
         # @rollback_count = 0
+        @enable_size_metrics = false
+      end
+
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'multi_output' => stats }
       end
 
       def multi_output?
@@ -60,6 +90,12 @@ module Fluent
       def configure(conf)
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+
         @stores.each do |store|
           store_conf = store.corresponding_config_element
           type = store_conf['@type']
@@ -143,12 +179,13 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/out_copy.rb

@@ -46,7 +46,7 @@ module Fluent::Plugin
         @ignore_errors << (store.arg.include?('ignore_error'))
         @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
       }
-      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
+      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
         log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
       end
     end

data/lib/fluent/plugin/out_exec_filter.rb

@@ -163,7 +163,7 @@ module Fluent::Plugin
                     0
                   elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
                     -1
-                  elsif @child_respawn =~ /^\d+$/
+                  elsif /^\d+$/.match?(@child_respawn)
                     @child_respawn.to_i
                   else
                     raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
@@ -187,7 +187,7 @@ module Fluent::Plugin
       @rr = 0
 
       exit_callback = ->(status){
-        c = @children.select{|child| child.pid == status.pid }.first
+        c = @children.find{|child| child.pid == status.pid }
         if c
           unless self.stopped?
             log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig

data/lib/fluent/plugin/out_file.rb

@@ -181,6 +181,13 @@ module Fluent::Plugin
       @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
       @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
       @need_lock = system_config.workers > 1
+
+      # https://github.com/fluent/fluentd/issues/3569
+      @need_ruby_on_macos_workaround = false
+      if @append && Fluent.macos?
+        condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
+        @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
+      end
     end
 
     def multi_workers_ready?
@@ -210,7 +217,13 @@ module Fluent::Plugin
                end
 
       if @append
-        writer.call(path, chunk)
+        if @need_lock
+          acquire_worker_lock(path) do
+            writer.call(path, chunk)
+          end
+        else
+          writer.call(path, chunk)
+        end
       else
         find_filepath_available(path, with_lock: @need_lock) do |actual_path|
           writer.call(actual_path, chunk)
@@ -223,7 +236,12 @@ module Fluent::Plugin
 
     def write_without_compression(path, chunk)
       File.open(path, "ab", @file_perm) do |f|
-        chunk.write_to(f)
+        if @need_ruby_on_macos_workaround
+          content = chunk.read()
+          f.puts content
+        else
+          chunk.write_to(f)
+        end
       end
     end
 

data/lib/fluent/plugin/out_forward/ack_handler.rb

@@ -59,7 +59,13 @@ module Fluent::Plugin
             @ack_waitings = new_list
           end
 
-          readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          begin
+            readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          rescue IOError
+            @log.info "connection closed while waiting for readable sockets"
+            readable_sockets = nil
+          end
+
           if readable_sockets
             readable_sockets.each do |sock|
               results << read_ack_from_sock(sock)
@@ -109,13 +115,22 @@ module Fluent::Plugin
           raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
         rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
           raw_data = ''
+        rescue IOError
+          @log.info "socket closed while receiving ack response"
+          return nil, Result::FAILED
         end
 
         info = find(sock)
 
-        # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-        # If this happens we assume the data wasn't delivered and retry it.
-        if raw_data.empty?
+        if info.nil?
+          # The info can be deleted by another thread during `sock.recv()` and `find()`.
+          # This is OK since another thread has completed to process the ack, so we can skip this.
+          # Note: exclusion mechanism about `collect_response()` may need to be considered.
+          @log.debug "could not find the ack info. this ack may be processed by another thread."
+          return nil, Result::FAILED
+        elsif raw_data.empty?
+          # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+          # If this happens we assume the data wasn't delivered and retry it.
           @log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
           # info.node.disable!
           return info, Result::FAILED

data/lib/fluent/plugin/out_forward/socket_cache.rb

@@ -50,6 +50,7 @@ module Fluent::Plugin
       def checkin(sock)
         @mutex.synchronize do
           if (s = @inflight_sockets.delete(sock))
+            s.timeout = timeout
             @available_sockets[s.key] << s
           else
             @log.debug("there is no socket #{sock}")
@@ -122,6 +123,7 @@ module Fluent::Plugin
         t = Time.now
         if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })
           @inflight_sockets[s.sock] = @available_sockets[key].delete(s)
+          s.timeout = timeout
           s
         else
           nil

data/lib/fluent/plugin/out_forward.rb

@@ -167,6 +167,8 @@ module Fluent::Plugin
       @usock = nil
       @keep_alive_watcher_interval = 5 # TODO
       @suspend_flush = false
+      @healthy_nodes_count_metrics = nil
+      @registered_nodes_count_metrics = nil
     end
 
     def configure(conf)
@@ -265,6 +267,9 @@ module Fluent::Plugin
       end
 
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
+      @healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
+      @registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
+
     end
 
     def multi_workers_ready?
@@ -418,18 +423,21 @@ module Fluent::Plugin
     def statistics
       stats = super
       services = service_discovery_services
-      healthy_nodes_count = 0
-      registed_nodes_count = services.size
+      @healthy_nodes_count_metrics.set(0)
+      @registered_nodes_count_metrics.set(services.size)
       services.each do |s|
         if s.available?
-          healthy_nodes_count += 1
+          @healthy_nodes_count_metrics.inc
         end
       end
 
-      stats.merge(
-        'healthy_nodes_count' => healthy_nodes_count,
-        'registered_nodes_count' => registed_nodes_count,
-      )
+      stats = {
+        'output' => stats["output"].merge({
+          'healthy_nodes_count' => @healthy_nodes_count_metrics.get,
+          'registered_nodes_count' => @registered_nodes_count_metrics.get,
+        })
+      }
+      stats
     end
 
     # MessagePack FixArray length is 3
@@ -513,8 +521,8 @@ module Fluent::Plugin
         when AckHandler::Result::SUCCESS
           commit_write(chunk_id)
         when AckHandler::Result::FAILED
-          node.disable!
-          rollback_write(chunk_id, update_retry: false)
+          node&.disable!
+          rollback_write(chunk_id, update_retry: false) if chunk_id
         when AckHandler::Result::CHUNKID_UNMATCHED
           rollback_write(chunk_id, update_retry: false)
         else