fluentd 1.13.3 → 1.16.5

data/lib/fluent/event_router.rb

@@ -47,6 +47,8 @@ module Fluent
       @match_cache = MatchCache.new
       @default_collector = default_collector
       @emit_error_handler = emit_error_handler
+      @metric_callbacks = {}
+      @caller_plugin_id = nil
     end
 
     attr_accessor :default_collector
@@ -83,6 +85,22 @@ module Fluent
       @match_rules << Rule.new(pattern, collector)
     end
 
+    def add_metric_callbacks(caller_plugin_id, callback)
+      @metric_callbacks[caller_plugin_id] = callback
+    end
+
+    def caller_plugin_id=(caller_plugin_id)
+      @caller_plugin_id = caller_plugin_id
+    end
+
+    def find_callback
+      if @caller_plugin_id
+        @metric_callbacks[@caller_plugin_id]
+      else
+        nil
+      end
+    end
+
     def emit(tag, time, record)
       unless record.nil?
         emit_stream(tag, OneEventStream.new(time, record))
@@ -95,6 +113,11 @@ module Fluent
 
     def emit_stream(tag, es)
       match(tag).emit_events(tag, es)
+      if callback = find_callback
+        callback.call(es)
+      end
+    rescue Pipeline::OutputError => e
+      @emit_error_handler.handle_emits_error(tag, e.processed_es, e.internal_error)
     rescue => e
       @emit_error_handler.handle_emits_error(tag, es, e)
     end
@@ -140,6 +163,17 @@ module Fluent
     private
 
     class Pipeline
+
+      class OutputError < StandardError
+        attr_reader :internal_error
+        attr_reader :processed_es
+
+        def initialize(internal_error, processed_es)
+          @internal_error = internal_error
+          @processed_es = processed_es
+        end
+      end
+
       def initialize
         @filters = []
         @output = nil
@@ -157,7 +191,12 @@ module Fluent
 
       def emit_events(tag, es)
         processed = @optimizer.filter_stream(tag, es)
-        @output.emit_events(tag, processed)
+
+        begin
+          @output.emit_events(tag, processed)
+        rescue => e
+          raise OutputError.new(e, processed)
+        end
       end
 
       class FilterOptimizer
@@ -175,7 +214,11 @@ module Fluent
           if optimizable?
             optimized_filter_stream(tag, es)
           else
-            @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
+            @filters.reduce(es) { |acc, filter|
+              filtered_es = filter.filter_stream(tag, acc)
+              filter.measure_metrics(filtered_es)
+              filtered_es
+            }
           end
         end
 
@@ -193,6 +236,7 @@ module Fluent
                   begin
                     filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
                     throw :break_loop unless filtered_record && filtered_time
+                    filter.measure_metrics(OneEventStream.new(time, record))
                   rescue => e
                     filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
                   end
@@ -200,6 +244,7 @@ module Fluent
                   begin
                     filtered_record = filter.filter(tag, filtered_time, filtered_record)
                     throw :break_loop unless filtered_record
+                    filter.measure_metrics(OneEventStream.new(time, record))
                   rescue => e
                     filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
                   end

data/lib/fluent/file_wrapper.rb

@@ -0,0 +1,137 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+unless Fluent.windows?
+  Fluent::FileWrapper = File
+else
+  require 'fluent/win32api'
+
+  module Fluent
+    module FileWrapper
+      def self.open(path, mode='r')
+        io = WindowsFile.new(path, mode).io
+        if block_given?
+          v = yield io
+          io.close
+          v
+        else
+          io
+        end
+      end
+
+      def self.stat(path)
+        f = WindowsFile.new(path)
+        s = f.stat
+        f.close
+        s
+      end
+    end
+
+    class WindowsFile
+      include File::Constants
+
+      attr_reader :io
+
+      INVALID_HANDLE_VALUE = -1
+
+      def initialize(path, mode_enc='r')
+        @path = path
+        mode, enc = mode_enc.split(":", 2)
+        @io = File.open(path, mode2flags(mode))
+        @io.set_encoding(enc) if enc
+        @file_handle = Win32API._get_osfhandle(@io.to_i)
+        @io.instance_variable_set(:@file_index, self.ino)
+        def @io.ino
+          @file_index
+        end
+      end
+
+      def close
+        @io.close
+        @file_handle = INVALID_HANDLE_VALUE
+      end
+
+      # To keep backward compatibility, we continue to use GetFileInformationByHandle()
+      # to get file id.
+      # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
+      # and returned value is different with above one, former one is 64 bit while
+      # later one is 128bit.
+      def ino
+        by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
+
+        unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
+          return 0
+        end
+
+        by_handle_file_information.unpack("I11Q1")[11] # fileindex
+      end
+
+      def stat
+        raise Errno::ENOENT if delete_pending
+        s = File.stat(@path)
+        s.instance_variable_set :@ino, self.ino
+        def s.ino; @ino; end
+        s
+      end
+
+      private
+
+      def mode2flags(mode)
+        # Always inject File::Constants::SHARE_DELETE
+        # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
+        # To enable SHARE_DELETE, BINARY is also required.
+        # https://bugs.ruby-lang.org/issues/11218
+        # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
+        flags = BINARY | SHARE_DELETE
+        case mode.delete("b")
+        when "r"
+          flags |= RDONLY
+        when "r+"
+          flags |= RDWR
+        when "w"
+          flags |= WRONLY | CREAT | TRUNC
+        when "w+"
+          flags |= RDWR | CREAT | TRUNC
+        when "a"
+          flags |= WRONLY | CREAT | APPEND
+        when "a+"
+          flags |= RDWR | CREAT | APPEND
+        else
+          raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
+        end
+      end
+
+      # DeletePending is a Windows-specific file state that roughly means
+      # "this file is queued for deletion, so close any open handlers"
+      #
+      # This flag can be retrieved via GetFileInformationByHandleEx().
+      #
+      # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+      #
+      def delete_pending
+        file_standard_info = 0x01
+        bufsize = 1024
+        buf = '\0' * bufsize
+
+        unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
+          return false
+        end
+
+        return buf.unpack("QQICC")[3] != 0
+      end
+    end
+  end
+end

data/lib/fluent/log/console_adapter.rb

@@ -0,0 +1,66 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'console/terminal/logger'
+
+module Fluent
+  class Log
+    # Async gem which is used by http_server helper switched logger mechanism to
+    # Console gem which isn't complatible with Ruby's standard Logger (since
+    # v1.17). This class adapts it to Fluentd's logger mechanism.
+    class ConsoleAdapter < Console::Terminal::Logger
+      def self.wrap(logger)
+        _, level = Console::Logger::LEVELS.find { |key, value|
+          if logger.level <= 0
+            key == :debug
+          else
+            value == logger.level - 1
+          end
+        }
+        Console::Logger.new(ConsoleAdapter.new(logger), level: level)
+      end
+
+      def initialize(logger)
+        @logger = logger
+        # When `verbose` is `true`, following items will be added as a prefix or
+        # suffix of the subject:
+        #   * Severity
+        #   * Object ID
+        #   * PID
+        #   * Time
+        # Severity and Time are added by Fluentd::Log too so they are redundant.
+        # PID is the worker's PID so it's also redundant.
+        # Object ID will be too verbose in usual cases.
+        # So set it as `false` here to suppress redundant items.
+        super(StringIO.new, verbose: false)
+      end
+
+      def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)
+        if LEVEL_TEXT.include?(severity.to_s)
+          level = severity
+        else
+          @logger.warn("Unknown severity: #{severity}")
+          level = 'warn'
+        end
+
+        @io.seek(0)
+        @io.truncate(0)
+        super
+        @logger.send(level, @io.string.chomp)
+      end
+    end
+  end
+end

data/lib/fluent/log.rb

@@ -51,6 +51,8 @@ module Fluent
     LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
     LOG_ROTATE_AGE = %w(daily weekly monthly)
 
+    IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
+
     def self.str_to_level(log_level_str)
       case log_level_str.downcase
       when "trace" then LEVEL_TRACE
@@ -67,8 +69,29 @@ module Fluent
       LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
     end
 
+    # Create a unique path for each process.
+    #
+    # >>> per_process_path("C:/tmp/test.log", :worker, 1)
+    # C:/tmp/test-1.log
+    # >>> per_process_path("C:/tmp/test.log", :supervisor, 0)
+    # C:/tmp/test-supervisor-0.log
+    def self.per_process_path(path, process_type, worker_id)
+      path = Pathname(path)
+      ext = path.extname
+
+      if process_type == :supervisor
+        suffix = "-#{process_type}-0#{ext}"  # "-0" for backword compatibility.
+      else
+        suffix = "-#{worker_id}#{ext}"
+      end
+      return path.sub_ext(suffix).to_s
+    end
+
     def initialize(logger, opts={})
-      # overwrites logger.level= so that config reloading resets level of Fluentd::Log
+      # When ServerEngine changes the logger.level, the Fluentd logger level should also change.
+      # So overwrites logger.level= below.
+      # However, currently Fluentd doesn't use the ServerEngine's reloading feature,
+      # so maybe we don't need this overwriting anymore.
       orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
       me = self
       # The original ruby logger sets the number as each log level like below.
@@ -92,6 +115,7 @@ module Fluent
       # So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.
       logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }
 
+      @path = opts[:path]
       @logger = logger
       @out = logger.instance_variable_get(:@logdev)
       @level = logger.level + 1
@@ -102,7 +126,8 @@ module Fluent
       @time_format = nil
       @formatter = nil
 
-      self.format = :text
+      self.format = opts.fetch(:format, :text)
+      self.time_format = opts[:time_format] if opts.key?(:time_format)
       enable_color out.tty?
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
@@ -154,8 +179,12 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval, :suppress_repeated_stacktrace
     attr_accessor :out
+    # Strictly speaking, we should also change @logger.level when the setter of @level is called.
+    # Currently, we don't need to do it, since Fluentd::Log doesn't use ServerEngine::DaemonLogger.level.
+    # Since We overwrites logger.level= so that @logger.level is applied to @level,
+    # we need to find a good way to do this, otherwise we will end up in an endless loop.
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
 
@@ -202,9 +231,12 @@ module Fluent
       @time_formatter = Strftime.new(@time_format) rescue nil
     end
 
+    def stdout?
+      @out == $stdout
+    end
+
     def reopen!
-      # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
-      @logger.reopen! if @logger
+      @out.reopen(@path, "a") if @path && @path != "-"
       nil
     end
 
@@ -447,6 +479,13 @@ module Fluent
           false
         end
       else
+        if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
+          cached_log.reject! do |_, cached_time|
+            (time - cached_time) > @ignore_same_log_interval
+          end
+        end
+        # If the size is still over, we have no choice but to clear it.
+        cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
         cached_log[message] = time
         false
       end

data/lib/fluent/match.rb

@@ -115,7 +115,7 @@ module Fluent
           stack.last << regex.pop
           regex.push ''
 
-        elsif c =~ /[a-zA-Z0-9_]/
+        elsif /[a-zA-Z0-9_]/.match?(c)
           regex.last << c
 
         else

data/lib/fluent/msgpack_factory.rb

@@ -100,7 +100,12 @@ module Fluent
     end
 
     def self.thread_local_msgpack_unpacker
-      Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
+      unpacker = Thread.current[:local_msgpack_unpacker]
+      if unpacker.nil?
+        return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
+      end
+      unpacker.reset
+      unpacker
     end
   end
 end

data/lib/fluent/oj_options.rb

@@ -4,14 +4,13 @@ module Fluent
   class OjOptions
     OPTIONS = {
       'bigdecimal_load': :symbol,
-      'max_nesting': :integer,
       'mode': :symbol,
       'use_to_json': :bool
     }
 
     ALLOWED_VALUES = {
       'bigdecimal_load': %i[bigdecimal float auto],
-      'mode': %i[strict null compat json rails object custom]
+      'mode': %i[strict null compat json rails custom]
     }
 
     DEFAULTS = {

data/lib/fluent/plugin/bare_output.rb

@@ -23,37 +23,78 @@ require 'fluent/plugin_helper'
 module Fluent
   module Plugin
     class BareOutput < Base
+      include PluginHelper::Mixin # for metrics
+
       # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
       # This output plugin base class is only for meta-output plugins
       # which cannot be implemented on MultiOutput.
       # E.g,: forest, config-expander
 
+      helpers_internal :metrics
+
       include PluginId
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      attr_reader :num_errors, :emit_count, :emit_records
-
       def process(tag, es)
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
       def initialize
         super
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+      end
+
+      def configure(conf)
+        super
+
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_count", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'bare_output' => stats }
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/base.rb

@@ -31,6 +31,7 @@ module Fluent
       def initialize
         @log = nil
         super
+        @fluentd_lock_dir = ENV['FLUENTD_LOCK_DIR']
         @_state = State.new(false, false, false, false, false, false, false, false, false)
         @_context_router = nil
         @_fluentd_worker_id = nil
@@ -52,14 +53,12 @@ module Fluent
       end
 
       def configure(conf)
-        if Fluent::Engine.supervisor_mode || (conf.respond_to?(:for_this_worker?) && conf.for_this_worker?)
-          workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
-                      conf.target_worker_ids.size
-                    else
-                      1
-                    end
-          system_config_override(workers: workers)
+        raise ArgumentError, "BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed." unless conf.is_a?(Fluent::Config::Element)
+
+        if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)
+          system_config_override(workers: conf.target_worker_ids.size)
         end
+
         super(conf, system_config.strict_config_value)
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true
@@ -70,10 +69,28 @@ module Fluent
         true
       end
 
+      def get_lock_path(name)
+        name = name.gsub(/[^a-zA-Z0-9]/, "_")
+        File.join(@fluentd_lock_dir, "fluentd-#{name}.lock")
+      end
+
+      def acquire_worker_lock(name)
+        if @fluentd_lock_dir.nil?
+          raise InvalidLockDirectory, "can't acquire lock because FLUENTD_LOCK_DIR isn't set"
+        end
+        lock_path = get_lock_path(name)
+        File.open(lock_path, "w") do |f|
+          f.flock(File::LOCK_EX)
+          yield
+        end
+        # Update access time to prevent tmpwatch from deleting a lock file.
+        FileUtils.touch(lock_path);
+      end
+
       def string_safe_encoding(str)
         unless str.valid_encoding?
-          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
           str = str.scrub('?')
+          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
         end
         yield str
       end
@@ -173,7 +190,7 @@ module Fluent
           # Thread::Backtrace::Location#path returns base filename or absolute path.
           # #absolute_path returns absolute_path always.
           # https://bugs.ruby-lang.org/issues/12159
-          if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+          if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
             return true
           end
         end

data/lib/fluent/plugin/buf_file.rb

@@ -39,8 +39,8 @@ module Fluent
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
 
-      config_param :file_permission, :string, default: nil # '0644'
-      config_param :dir_permission,  :string, default: nil # '0755'
+      config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
+      config_param :dir_permission,  :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
 
       def initialize
         super
@@ -139,13 +139,20 @@ module Fluent
       def resume
         stage = {}
         queue = []
+        exist_broken_file = false
 
         patterns = [@path]
         patterns.unshift @additional_resume_path if @additional_resume_path
         Dir.glob(escaped_patterns(patterns)) do |path|
           next unless File.file?(path)
 
-          log.debug { "restoring buffer file: path = #{path}" }
+          if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
+            # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
+            # since there may be a power failure or similar failure.
+            log.warn { "restoring buffer file: path = #{path}" }
+          else
+            log.debug { "restoring buffer file: path = #{path}" }
+          end
 
           m = new_metadata() # this metadata will be overwritten by resuming .meta file content
                              # so it should not added into @metadata_list for now
@@ -158,6 +165,7 @@ module Fluent
           begin
             chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
           rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
+            exist_broken_file = true
             handle_broken_files(path, mode, e)
             next
           end
@@ -182,6 +190,15 @@ module Fluent
 
         queue.sort_by!{ |chunk| chunk.modified_at }
 
+        # If one of the files is corrupted, other files may also be corrupted and be undetected.
+        # The time priods of each chunk are helpful to check the data.
+        if exist_broken_file
+          log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
+          (stage.values + queue).each { |chunk|
+            log.info "  #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
+          }
+        end
+
         return stage, queue
       end
 
@@ -195,8 +212,20 @@ module Fluent
       end
 
       def handle_broken_files(path, mode, e)
-        log.error "found broken chunk file during resume. Deleted corresponding files:", :path => path, :mode => mode, :err_msg => e.message
-        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
+        unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)
+        backup(unique_id) { |f|
+          File.open(path, 'rb') { |chunk|
+            chunk.set_encoding(Encoding::ASCII_8BIT)
+            chunk.sync = true
+            chunk.binmode
+            IO.copy_stream(chunk, f)
+          }
+        }
+      rescue => error
+        log.error "backup failed. Delete corresponding files.", :err_msg => error.message
+      ensure
+        log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
         File.unlink(path, path + '.meta') rescue nil
       end