fluentd 1.20.0-aarch64-mingw-ucrt

data/lib/fluent/engine.rb

@@ -0,0 +1,269 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/event'
+require 'fluent/event_router'
+require 'fluent/msgpack_factory'
+require 'fluent/root_agent'
+require 'fluent/time'
+require 'fluent/system_config'
+require 'fluent/plugin'
+require 'fluent/fluent_log_event_router'
+require 'fluent/static_config_analysis'
+
+module Fluent
+  class EngineClass
+    # For compat. remove it in fluentd v2
+    include Fluent::MessagePackFactory::Mixin
+
+    def initialize
+      @root_agent = nil
+      @engine_stopped = false
+      @_worker_id = nil
+
+      @log_event_verbose = false
+      @suppress_config_dump = false
+      @without_source = false
+
+      @fluent_log_event_router = nil
+      @system_config = SystemConfig.new
+
+      @supervisor_mode = false
+
+      @root_agent_mutex = Mutex.new
+    end
+
+    MAINLOOP_SLEEP_INTERVAL = 0.3
+
+    attr_reader :root_agent, :system_config, :supervisor_mode
+
+    def init(system_config, supervisor_mode: false, start_in_parallel: false)
+      @system_config = system_config
+      @supervisor_mode = supervisor_mode
+
+      @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
+      @without_source = system_config.without_source unless system_config.without_source.nil?
+
+      @log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
+
+      @root_agent = RootAgent.new(log: log, system_config: @system_config, start_in_parallel: start_in_parallel)
+
+      self
+    end
+
+    def log
+      $log
+    end
+
+    def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
+      if /\.rb$/.match?(fname)
+        require 'fluent/config/dsl'
+        Config::DSL::Parser.parse(io, File.join(basepath, fname))
+      else
+        Config.parse(io, fname, basepath, v1_config)
+      end
+    end
+
+    def run_configure(conf, dry_run: false)
+      configure(conf)
+      conf.check_not_fetched do |key, e|
+        parent_name, plugin_name = e.unused_in
+        message = if parent_name && plugin_name
+                    "section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
+                  elsif parent_name
+                    "section <#{e.name}> is not used in <#{parent_name}>"
+                  elsif e.name != 'system' && !(@without_source && e.name == 'source')
+                    "parameter '#{key}' in #{e.to_s.strip} is not used."
+                  else
+                    nil
+                  end
+        next if message.nil?
+
+        if dry_run && @supervisor_mode
+          $log.warn :supervisor, message
+        elsif e.for_every_workers?
+          $log.warn :worker0, message
+        elsif e.for_this_worker?
+          $log.warn message
+        end
+      end
+    end
+
+    def configure(conf)
+      @root_agent.configure(conf)
+
+      @fluent_log_event_router = FluentLogEventRouter.build(@root_agent)
+
+      if @fluent_log_event_router.emittable?
+        $log.enable_event(true)
+      end
+
+      unless @suppress_config_dump
+        $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
+      end
+    end
+
+    def add_plugin_dir(dir)
+      $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::Plugin.add_plugin_dir')
+      Plugin.add_plugin_dir(dir)
+    end
+
+    def emit(tag, time, record)
+      raise "BUG: use router.emit instead of Engine.emit"
+    end
+
+    def emit_array(tag, array)
+      raise "BUG: use router.emit_array instead of Engine.emit_array"
+    end
+
+    def emit_stream(tag, es)
+      raise "BUG: use router.emit_stream instead of Engine.emit_stream"
+    end
+
+    def flush!
+      @root_agent_mutex.synchronize do
+        @root_agent.flush!
+      end
+    end
+
+    def cancel_source_only!
+      @root_agent_mutex.synchronize do
+        @root_agent.cancel_source_only!
+      end
+    end
+
+    def now
+      # TODO thread update
+      Fluent::EventTime.now
+    end
+
+    def run
+      begin
+        $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
+        @root_agent_mutex.synchronize do
+          start
+        end
+
+        @fluent_log_event_router.start
+
+        $log.info "fluentd worker is now running", worker: worker_id
+        sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
+        $log.info "fluentd worker is now stopping", worker: worker_id
+
+      rescue Exception => e
+        $log.error "unexpected error", error: e
+        $log.error_backtrace
+        raise
+      end
+
+      @root_agent_mutex.synchronize do
+        stop_phase(@root_agent)
+      end
+    end
+
+    # @param conf [Fluent::Config]
+    # @param supervisor [Bool]
+    # @return nil
+    def reload_config(conf, supervisor: false)
+      @root_agent_mutex.synchronize do
+        # configure first to reduce down time while restarting
+        new_agent = RootAgent.new(log: log, system_config: @system_config)
+        ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
+
+        ret.all_plugins.each do |plugin|
+          if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
+            raise Fluent::ConfigError, "Unreloadable plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
+          end
+        end
+
+        # Assign @root_agent to new root_agent
+        # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
+        old_agent, @root_agent = @root_agent, new_agent
+        begin
+          @root_agent.configure(conf)
+        rescue
+          @root_agent = old_agent
+          raise
+        end
+
+        unless @suppress_config_dump
+          $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
+        end
+
+        # supervisor doesn't handle actual data. so the following code is unnecessary.
+        if supervisor
+          old_agent.shutdown      # to close thread created in #configure
+          return
+        end
+
+        stop_phase(old_agent)
+
+        $log.info 'restart fluentd worker', worker: worker_id
+        start_phase(new_agent)
+      end
+    end
+
+    def stop
+      @engine_stopped = true
+      nil
+    end
+
+    def push_log_event(tag, time, record)
+      @fluent_log_event_router.emit_event([tag, time, record])
+    end
+
+    def worker_id
+      if @supervisor_mode
+        return -1
+      end
+
+      return @_worker_id if @_worker_id
+      # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
+      # so it's (almost) a single worker, worker_id=0
+      @_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+      @_worker_id
+    end
+
+    private
+
+    def stop_phase(root_agent)
+      unless @log_event_verbose
+        $log.enable_event(false)
+        @fluent_log_event_router.graceful_stop
+      end
+      $log.info 'shutting down fluentd worker', worker: worker_id
+      root_agent.shutdown
+
+      @fluent_log_event_router.stop
+    end
+
+    def start_phase(root_agent)
+      @fluent_log_event_router = FluentLogEventRouter.build(root_agent)
+      if @fluent_log_event_router.emittable?
+        $log.enable_event(true)
+      end
+
+      @root_agent.start
+    end
+
+    def start
+      @root_agent.start
+    end
+  end
+
+  Engine = EngineClass.new
+end

data/lib/fluent/env.rb

@@ -0,0 +1,45 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'securerandom'
+
+require 'serverengine/utils'
+require 'fluent/oj_options'
+
+module Fluent
+  DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
+  DEFAULT_CONFIG_INCLUDE_DIR = ENV["FLUENT_CONF_INCLUDE_DIR"] || '/etc/fluent/conf.d'
+  DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
+  DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+  DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
+  DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
+  DEFAULT_JSON_PARSE_OPTIONS = { allow_duplicate_key: true, allow_comments: true }.freeze
+  DEFAULT_DIR_PERMISSION = 0755
+  DEFAULT_FILE_PERMISSION = 0644
+  INSTANCE_ID = ENV['FLUENT_INSTANCE_ID'] || SecureRandom.uuid
+
+  def self.windows?
+    ServerEngine.windows?
+  end
+
+  def self.linux?
+    RUBY_PLATFORM.include?("linux")
+  end
+
+  def self.macos?
+    RUBY_PLATFORM.include?("darwin")
+  end
+end

data/lib/fluent/error.rb

@@ -0,0 +1,37 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class UnrecoverableError < StandardError
+    def initialize(error_message = nil)
+      @message = error_message || "an unrecoverable error occurs in Fluentd process"
+    end
+
+    def to_s
+      @message
+    end
+  end
+
+  class InvalidRootDirectory < UnrecoverableError
+  end
+
+  class InvalidLockDirectory < UnrecoverableError
+  end
+
+  # For internal use
+  class UncatchableError < Exception
+  end
+end

data/lib/fluent/event.rb

@@ -0,0 +1,369 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/msgpack_factory'
+require 'fluent/plugin/compressable'
+
+module Fluent
+  class EventStream
+    include Enumerable
+    include Fluent::Plugin::Compressable
+
+    # dup does deep copy for event stream
+    def dup
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
+    def size
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+    alias :length :size
+
+    def empty?
+      size == 0
+    end
+
+    # for tests
+    def ==(other)
+      other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream
+    end
+
+    def repeatable?
+      false
+    end
+
+    def slice(index, num)
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
+    def each(unpacker: nil, &block)
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
+    def to_msgpack_stream(time_int: false, packer: nil)
+      return to_msgpack_stream_forced_integer(packer: packer) if time_int
+      out = packer || Fluent::MessagePackFactory.msgpack_packer
+      each {|time,record|
+        out.write([time,record])
+      }
+      out.full_pack
+    end
+
+    def to_compressed_msgpack_stream(time_int: false, packer: nil, type: :gzip)
+      packed = to_msgpack_stream(time_int: time_int, packer: packer)
+      compress(packed, type: type)
+    end
+
+    def to_msgpack_stream_forced_integer(packer: nil)
+      out = packer || Fluent::MessagePackFactory.msgpack_packer
+      each {|time,record|
+        out.write([time.to_i,record])
+      }
+      out.full_pack
+    end
+  end
+
+  class OneEventStream < EventStream
+    def initialize(time, record)
+      @time = time
+      @record = record
+    end
+
+    def dup
+      OneEventStream.new(@time, @record.dup)
+    end
+
+    def empty?
+      false
+    end
+
+    def size
+      1
+    end
+
+    def repeatable?
+      true
+    end
+
+    def slice(index, num)
+      if index > 0 || num == 0
+        ArrayEventStream.new([])
+      else
+        self.dup
+      end
+    end
+
+    def each(unpacker: nil, &block)
+      block.call(@time, @record)
+      nil
+    end
+  end
+
+  # EventStream from entries: Array of [time, record]
+  #
+  # Use this class for many events data with a tag
+  # and its representation is [ [time, record], [time, record], .. ]
+  class ArrayEventStream < EventStream
+    def initialize(entries)
+      @entries = entries
+    end
+
+    def dup
+      entries = @entries.map{ |time, record| [time, record.dup] }
+      ArrayEventStream.new(entries)
+    end
+
+    def size
+      @entries.size
+    end
+
+    def repeatable?
+      true
+    end
+
+    def empty?
+      @entries.empty?
+    end
+
+    def slice(index, num)
+      ArrayEventStream.new(@entries.slice(index, num))
+    end
+
+    def each(unpacker: nil, &block)
+      @entries.each(&block)
+      nil
+    end
+  end
+
+  # EventStream from entries: numbers of pairs of time and record.
+  #
+  # This class can handle many events more efficiently than ArrayEventStream
+  # because this class generate less objects than ArrayEventStream.
+  #
+  # Use this class as below, in loop of data-enumeration:
+  #  1. initialize blank stream:
+  #     streams[tag] ||= MultiEventStream.new
+  #  2. add events
+  #     stream[tag].add(time, record)
+  class MultiEventStream < EventStream
+    def initialize(time_array = [], record_array = [])
+      @time_array = time_array
+      @record_array = record_array
+    end
+
+    def dup
+      MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))
+    end
+
+    def size
+      @time_array.size
+    end
+
+    def add(time, record)
+      @time_array << time
+      @record_array << record
+    end
+
+    def repeatable?
+      true
+    end
+
+    def empty?
+      @time_array.empty?
+    end
+
+    def slice(index, num)
+      MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
+    end
+
+    def each(unpacker: nil, &block)
+      time_array = @time_array
+      record_array = @record_array
+      for i in 0..time_array.length-1
+        block.call(time_array[i], record_array[i])
+      end
+      nil
+    end
+  end
+
+  class MessagePackEventStream < EventStream
+    # https://github.com/msgpack/msgpack-ruby/issues/119
+
+    # Keep cached_unpacker argument for existing plugins
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
+      @data = data
+      @size = size
+      @unpacked_times = unpacked_times
+      @unpacked_records = unpacked_records
+    end
+
+    def empty?
+      @data.empty?
+    end
+
+    def dup
+      if @unpacked_times
+        self.class.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
+      else
+        self.class.new(@data.dup, nil, @size)
+      end
+    end
+
+    def size
+      # @size is unbelievable always when @size == 0
+      # If the number of events is really zero, unpacking events takes very short time.
+      ensure_unpacked! if @size == 0
+      @size
+    end
+
+    def repeatable?
+      true
+    end
+
+    def ensure_unpacked!(unpacker: nil)
+      return if @unpacked_times && @unpacked_records
+      @unpacked_times = []
+      @unpacked_records = []
+      (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
+        @unpacked_times << time
+        @unpacked_records << record
+      end
+      # @size should be updated always right after unpack.
+      # The real size of unpacked objects are correct, rather than given size.
+      @size = @unpacked_times.size
+    end
+
+    # This method returns MultiEventStream, because there are no reason
+    # to survey binary serialized by msgpack.
+    def slice(index, num)
+      ensure_unpacked!
+      MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
+    end
+
+    def each(unpacker: nil, &block)
+      ensure_unpacked!(unpacker: unpacker)
+      @unpacked_times.each_with_index do |time, i|
+        block.call(time, @unpacked_records[i])
+      end
+      nil
+    end
+
+    def to_msgpack_stream(time_int: false, packer: nil)
+      # time_int is always ignored because @data is always packed binary in this class
+      @data
+    end
+  end
+
+  class CompressedMessagePackEventStream < MessagePackEventStream
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil, compress: :gzip, decompression_size_limit: Fluent::Plugin::Compressable::DEFAULT_DECOMPRESSION_SIZE_LIMIT)
+      super(data, cached_unpacker, size, unpacked_times: unpacked_times, unpacked_records: unpacked_records)
+      @decompressed_data = nil
+      @compressed_data = data
+      @type = compress
+      @decompression_size_limit = decompression_size_limit
+    end
+
+    def empty?
+      ensure_decompressed!
+      super
+    end
+
+    def ensure_unpacked!(unpacker: nil)
+      ensure_decompressed!
+      super
+    end
+
+    def each(unpacker: nil, &block)
+      ensure_decompressed!
+      super
+    end
+
+    def to_msgpack_stream(time_int: false, packer: nil)
+      ensure_decompressed!
+      super
+    end
+
+    def to_compressed_msgpack_stream(time_int: false, packer: nil)
+      # time_int is always ignored because @data is always packed binary in this class
+      @compressed_data
+    end
+
+    private
+
+    def ensure_decompressed!
+      return if @decompressed_data
+      @data = @decompressed_data = decompress(@data, type: @type)
+    end
+  end
+
+  class MetadataTimeEventStream < EventStream
+    def initialize(es)
+      @es = es
+    end
+
+    def empty?
+      @es.empty?
+    end
+
+    def dup
+      self.class.new(@es.dup)
+    end
+
+    def size
+      @es.size
+    end
+
+    def repeatable?
+      @es.repeatable?
+    end
+
+    def slice(index, num)
+      self.class.new(@es.slice(index, num))
+    end
+
+    def each(unpacker: nil, &block)
+      @es.each(unpacker: unpacker) do |time, record|
+        next if record.nil?
+
+        time = time[0] if time.is_a?(Array)
+        time = Fluent::EventTime.now if time.nil? || time.to_i == 0
+        block.call(time, record)
+      end
+      nil
+    end
+  end
+
+  module ChunkMessagePackEventStreamer
+    # chunk.extend(ChunkMessagePackEventStreamer)
+    #  => chunk.each{|time, record| ... }
+    def each(unpacker: nil, &block)
+      # Note: If need to use `unpacker`, then implement it,
+      # e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
+      raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
+
+      open do |io|
+        Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
+      end
+      nil
+    end
+    alias :msgpack_each :each
+
+    def to_msgpack_stream(time_int: false, packer: nil)
+      # time_int is always ignored because data is already packed and written in chunk
+      read
+    end
+  end
+end