fluentd222 1.16.2-x86_64-linux

data/lib/fluent/plugin/base.rb

@@ -0,0 +1,214 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/configurable'
+require 'fluent/system_config'
+
+module Fluent
+  module Plugin
+    class Base
+      include Configurable
+      include SystemConfig::Mixin
+
+      State = Struct.new(:configure, :start, :after_start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
+
+      attr_accessor :under_plugin_development
+
+      def initialize
+        @log = nil
+        super
+        @fluentd_lock_dir = ENV['FLUENTD_LOCK_DIR']
+        @_state = State.new(false, false, false, false, false, false, false, false, false)
+        @_context_router = nil
+        @_fluentd_worker_id = nil
+        @under_plugin_development = false
+      end
+
+      def has_router?
+        false
+      end
+
+      def plugin_root_dir
+        nil # override this in plugin_id.rb
+      end
+
+      def fluentd_worker_id
+        return @_fluentd_worker_id if @_fluentd_worker_id
+        @_fluentd_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+        @_fluentd_worker_id
+      end
+
+      def configure(conf)
+        raise ArgumentError, "BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed." unless conf.is_a?(Fluent::Config::Element)
+
+        if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)
+          system_config_override(workers: conf.target_worker_ids.size)
+        end
+
+        super(conf, system_config.strict_config_value)
+        @_state ||= State.new(false, false, false, false, false, false, false, false, false)
+        @_state.configure = true
+        self
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def get_lock_path(name)
+        name = name.gsub(/[^a-zA-Z0-9]/, "_")
+        File.join(@fluentd_lock_dir, "fluentd-#{name}.lock")
+      end
+
+      def acquire_worker_lock(name)
+        if @fluentd_lock_dir.nil?
+          raise InvalidLockDirectory, "can't acquire lock because FLUENTD_LOCK_DIR isn't set"
+        end
+        lock_path = get_lock_path(name)
+        File.open(lock_path, "w") do |f|
+          f.flock(File::LOCK_EX)
+          yield
+        end
+        # Update access time to prevent tmpwatch from deleting a lock file.
+        FileUtils.touch(lock_path);
+      end
+
+      def string_safe_encoding(str)
+        unless str.valid_encoding?
+          str = str.scrub('?')
+          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
+        end
+        yield str
+      end
+
+      def context_router=(router)
+        @_context_router = router
+      end
+
+      def context_router
+        @_context_router
+      end
+
+      def start
+        # By initialization order, plugin logger is created before set log_event_enabled.
+        # It causes '@id' specified plugin, it uses plugin logger instead of global logger, ignores `<label @FLUENT_LOG>` setting.
+        # This is adhoc approach but impact is minimal.
+        if @log.is_a?(Fluent::PluginLogger) && $log.respond_to?(:log_event_enabled) # log_event_enabled check for tests
+          @log.log_event_enabled = $log.log_event_enabled
+        end
+        @_state.start = true
+        self
+      end
+
+      def after_start
+        @_state.after_start = true
+        self
+      end
+
+      def stop
+        @_state.stop = true
+        self
+      end
+
+      def before_shutdown
+        @_state.before_shutdown = true
+        self
+      end
+
+      def shutdown
+        @_state.shutdown = true
+        self
+      end
+
+      def after_shutdown
+        @_state.after_shutdown = true
+        self
+      end
+
+      def close
+        @_state.close = true
+        self
+      end
+
+      def terminate
+        @_state.terminate = true
+        self
+      end
+
+      def configured?
+        @_state.configure
+      end
+
+      def started?
+        @_state.start
+      end
+
+      def after_started?
+        @_state.after_start
+      end
+
+      def stopped?
+        @_state.stop
+      end
+
+      def before_shutdown?
+        @_state.before_shutdown
+      end
+
+      def shutdown?
+        @_state.shutdown
+      end
+
+      def after_shutdown?
+        @_state.after_shutdown
+      end
+
+      def closed?
+        @_state.close
+      end
+
+      def terminated?
+        @_state.terminate
+      end
+
+      def called_in_test?
+        caller_locations.each do |location|
+          # Thread::Backtrace::Location#path returns base filename or absolute path.
+          # #absolute_path returns absolute_path always.
+          # https://bugs.ruby-lang.org/issues/12159
+          if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
+            return true
+          end
+        end
+        false
+      end
+
+      def inspect
+        # Plugin instances are sometimes too big to dump because it may have too many thins (buffer,storage, ...)
+        # Original commit comment says that:
+        #   To emulate normal inspect behavior `ruby -e'o=Object.new;p o;p (o.__id__<<1).to_s(16)'`.
+        #   https://github.com/ruby/ruby/blob/trunk/gc.c#L788
+        "#<%s:%014x>" % [self.class.name, '0x%014x' % (__id__ << 1)]
+      end
+
+      def reloadable_plugin?
+        # Engine can't capture all class variables. so it's forbbiden to use class variables in each plugins if enabling reload.
+        self.class.class_variables.empty?
+      end
+    end
+  end
+end

data/lib/fluent/plugin/buf_file.rb

@@ -0,0 +1,242 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fileutils'
+
+require 'fluent/plugin/buffer'
+require 'fluent/plugin/buffer/file_chunk'
+require 'fluent/system_config'
+require 'fluent/variable_store'
+
+module Fluent
+  module Plugin
+    class FileBuffer < Fluent::Plugin::Buffer
+      Plugin.register_buffer('file', self)
+
+      include SystemConfig::Mixin
+
+      DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024        # 256MB
+      DEFAULT_TOTAL_LIMIT_SIZE =  64 * 1024 * 1024 * 1024 #  64GB, same with v0.12 (TimeSlicedOutput + buf_file)
+
+      desc 'The path where buffer chunks are stored.'
+      config_param :path, :string, default: nil
+      desc 'The suffix of buffer chunks'
+      config_param :path_suffix, :string, default: '.log'
+
+      config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
+      config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
+
+      config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
+      config_param :dir_permission,  :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
+
+      def initialize
+        super
+        @symlink_path = nil
+        @multi_workers_available = false
+        @additional_resume_path = nil
+        @buffer_path = nil
+        @variable_store = nil
+      end
+
+      def configure(conf)
+        super
+
+        @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
+
+        multi_workers_configured = owner.system_config.workers > 1
+
+        using_plugin_root_dir = false
+        unless @path
+          if root_dir = owner.plugin_root_dir
+            @path = File.join(root_dir, 'buffer')
+            using_plugin_root_dir = true # plugin_root_dir path contains worker id
+          else
+            raise Fluent::ConfigError, "buffer path is not configured. specify 'path' in <buffer>"
+          end
+        end
+
+        type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
+        if @variable_store.has_key?(@path) && !called_in_test?
+          type_using_this_path = @variable_store[@path]
+          raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
+        end
+
+        @buffer_path = @path
+        @variable_store[@buffer_path] = type_of_owner
+
+        specified_directory_exists = File.exist?(@path) && File.directory?(@path)
+        unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
+
+        if specified_directory_exists || unexisting_path_for_directory # directory
+          if using_plugin_root_dir || !multi_workers_configured
+            @path = File.join(@path, "buffer.*#{@path_suffix}")
+          else
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "buffer.*#{@path_suffix}")
+            if fluentd_worker_id == 0
+              # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
+              @additional_resume_path = File.join(File.expand_path("../../", @path), "buffer.*#{@path_suffix}")
+            end
+          end
+          @multi_workers_available = true
+        else # specified path is file path
+          if File.basename(@path).include?('.*.')
+            # valid file path
+          elsif File.basename(@path).end_with?('.*')
+            @path = @path + @path_suffix
+          else
+            # existing file will be ignored
+            @path = @path + ".*#{@path_suffix}"
+          end
+          @multi_workers_available = false
+        end
+
+        if @dir_permission
+          @dir_permission = @dir_permission.to_i(8) if @dir_permission.is_a?(String)
+        else
+          @dir_permission = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
+        end
+      end
+
+      # This method is called only when multi worker is configured
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "file buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
+      end
+
+      def start
+        FileUtils.mkdir_p File.dirname(@path), mode: @dir_permission
+
+        super
+      end
+
+      def stop
+        if @variable_store
+          @variable_store.delete(@buffer_path)
+        end
+
+        super
+      end
+
+      def persistent?
+        true
+      end
+
+      def resume
+        stage = {}
+        queue = []
+        exist_broken_file = false
+
+        patterns = [@path]
+        patterns.unshift @additional_resume_path if @additional_resume_path
+        Dir.glob(escaped_patterns(patterns)) do |path|
+          next unless File.file?(path)
+
+          if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
+            # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
+            # since there may be a power failure or similar failure.
+            log.warn { "restoring buffer file: path = #{path}" }
+          else
+            log.debug { "restoring buffer file: path = #{path}" }
+          end
+
+          m = new_metadata() # this metadata will be overwritten by resuming .meta file content
+                             # so it should not added into @metadata_list for now
+          mode = Fluent::Plugin::Buffer::FileChunk.assume_chunk_state(path)
+          if mode == :unknown
+            log.debug "unknown state chunk found", path: path
+            next
+          end
+
+          begin
+            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
+          rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
+            exist_broken_file = true
+            handle_broken_files(path, mode, e)
+            next
+          end
+
+          case chunk.state
+          when :staged
+            # unstaged chunk created at Buffer#write_step_by_step is identified as the staged chunk here because FileChunk#assume_chunk_state checks only the file name.
+            # https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L663
+            # This case can happen when fluentd process is killed by signal or other reasons between creating unstaged chunks and changing them to staged mode in Buffer#write
+            # these chunks(unstaged chunks) has shared the same metadata
+            # So perform enqueue step again https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L364
+            if chunk_size_full?(chunk) || stage.key?(chunk.metadata)
+              chunk.metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
+              queue << chunk.enqueued!
+            else
+              stage[chunk.metadata] = chunk
+            end
+          when :queued
+            queue << chunk
+          end
+        end
+
+        queue.sort_by!{ |chunk| chunk.modified_at }
+
+        # If one of the files is corrupted, other files may also be corrupted and be undetected.
+        # The time priods of each chunk are helpful to check the data.
+        if exist_broken_file
+          log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
+          (stage.values + queue).each { |chunk|
+            log.info "  #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
+          }
+        end
+
+        return stage, queue
+      end
+
+      def generate_chunk(metadata)
+        # FileChunk generates real path with unique_id
+        perm = @file_permission || system_config.file_permission
+        chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: perm, compress: @compress)
+        log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
+
+        return chunk
+      end
+
+      def handle_broken_files(path, mode, e)
+        log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
+        unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)
+        backup(unique_id) { |f|
+          File.open(path, 'rb') { |chunk|
+            chunk.set_encoding(Encoding::ASCII_8BIT)
+            chunk.sync = true
+            chunk.binmode
+            IO.copy_stream(chunk, f)
+          }
+        }
+      rescue => error
+        log.error "backup failed. Delete corresponding files.", :err_msg => error.message
+      ensure
+        log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
+        File.unlink(path, path + '.meta') rescue nil
+      end
+
+      private
+
+      def escaped_patterns(patterns)
+        patterns.map { |pattern|
+          # '{' '}' are special character in Dir.glob
+          pattern.gsub(/[\{\}]/) { |c| "\\#{c}" }
+        }
+      end
+    end
+  end
+end