fluentd 1.20.0-aarch64-mingw-ucrt

data/lib/fluent/event_router.rb

@@ -0,0 +1,315 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/match'
+require 'fluent/event'
+require 'fluent/filter'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  #
+  # EventRouter is responsible to route events to a collector.
+  #
+  # It has a list of MatchPattern and Collector pairs:
+  #
+  #  +----------------+     +-----------------+
+  #  |  MatchPattern  |     |    Collector    |
+  #  +----------------+     +-----------------+
+  #  |   access.**  ---------> type forward   |
+  #  |     logs.**  ---------> type copy      |
+  #  |  archive.**  ---------> type s3        |
+  #  +----------------+     +-----------------+
+  #
+  # EventRouter does:
+  #
+  # 1) receive an event at `#emit` methods
+  # 2) match the event's tag with the MatchPatterns
+  # 3) forward the event to the corresponding Collector
+  #
+  # Collector is either of Output, Filter or other EventRouter.
+  #
+  class EventRouter
+    def initialize(default_collector, emit_error_handler)
+      @match_rules = []
+      @match_cache = MatchCache.new
+      @default_collector = default_collector
+      @emit_error_handler = emit_error_handler
+      @metric_callbacks = {}
+      @caller_plugin_id = nil
+    end
+
+    attr_accessor :default_collector
+    attr_accessor :emit_error_handler
+
+    class Rule
+      def initialize(pattern, collector)
+        patterns = pattern.split(/\s+/).map { |str| MatchPattern.create(str) }
+        @pattern = if patterns.length == 1
+                     patterns[0]
+                   else
+                     OrMatchPattern.new(patterns)
+                   end
+        @pattern_str = pattern
+        @collector = collector
+      end
+
+      def match?(tag)
+        @pattern.match(tag)
+      end
+
+      attr_reader :collector
+      attr_reader :pattern_str
+    end
+
+    def suppress_missing_match!
+      if @default_collector.respond_to?(:suppress_missing_match!)
+        @default_collector.suppress_missing_match!
+      end
+    end
+
+    # called by Agent to add new match pattern and collector
+    def add_rule(pattern, collector)
+      @match_rules << Rule.new(pattern, collector)
+    end
+
+    def add_metric_callbacks(caller_plugin_id, callback)
+      @metric_callbacks[caller_plugin_id] = callback
+    end
+
+    def caller_plugin_id=(caller_plugin_id)
+      @caller_plugin_id = caller_plugin_id
+    end
+
+    def find_callback
+      if @caller_plugin_id
+        @metric_callbacks[@caller_plugin_id]
+      else
+        nil
+      end
+    end
+
+    def emit(tag, time, record)
+      unless record.nil?
+        emit_stream(tag, OneEventStream.new(time, record))
+      end
+    end
+
+    def emit_array(tag, array)
+      emit_stream(tag, ArrayEventStream.new(array))
+    end
+
+    def emit_stream(tag, es)
+      match(tag).emit_events(tag, es)
+      if callback = find_callback
+        callback.call(es)
+      end
+    rescue Pipeline::OutputError => e
+      @emit_error_handler.handle_emits_error(tag, e.processed_es, e.internal_error)
+    rescue => e
+      @emit_error_handler.handle_emits_error(tag, es, e)
+    end
+
+    def emit_error_event(tag, time, record, error)
+      @emit_error_handler.emit_error_event(tag, time, record, error)
+    end
+
+    def match?(tag)
+      !!find(tag)
+    end
+
+    def match(tag)
+      collector = @match_cache.get(tag) {
+        find(tag) || @default_collector
+      }
+      collector
+    end
+
+    class MatchCache
+      MATCH_CACHE_SIZE = 1024
+
+      def initialize
+        super
+        @map = {}
+        @keys = []
+      end
+
+      def get(key)
+        if collector = @map[key]
+          return collector
+        end
+        collector = @map[key] = yield
+        if @keys.size >= MATCH_CACHE_SIZE
+          # expire the oldest key
+          @map.delete @keys.shift
+        end
+        @keys << key
+        collector
+      end
+    end
+
+    private
+
+    class Pipeline
+
+      class OutputError < StandardError
+        attr_reader :internal_error
+        attr_reader :processed_es
+
+        def initialize(internal_error, processed_es)
+          @internal_error = internal_error
+          @processed_es = processed_es
+        end
+      end
+
+      def initialize
+        @filters = []
+        @output = nil
+        @optimizer = FilterOptimizer.new
+      end
+
+      def add_filter(filter)
+        @filters << filter
+        @optimizer.filters = @filters
+      end
+
+      def set_output(output)
+        @output = output
+      end
+
+      def emit_events(tag, es)
+        processed = @optimizer.filter_stream(tag, es)
+
+        begin
+          @output.emit_events(tag, processed)
+        rescue => e
+          raise OutputError.new(e, processed)
+        end
+      end
+
+      class FilterOptimizer
+        def initialize(filters = [])
+          @filters = filters
+          @optimizable = nil
+        end
+
+        def filters=(filters)
+          @filters = filters
+          reset_optimization
+        end
+
+        def filter_stream(tag, es)
+          if optimizable?
+            optimized_filter_stream(tag, es)
+          else
+            @filters.reduce(es) { |acc, filter|
+              filtered_es = filter.filter_stream(tag, acc)
+              filter.measure_metrics(filtered_es)
+              filtered_es
+            }
+          end
+        end
+
+        private
+
+        def optimized_filter_stream(tag, es)
+          new_es = MultiEventStream.new
+          es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
+            filtered_record = record
+            filtered_time = time
+
+            catch :break_loop do
+              @filters.each do |filter|
+                if filter.has_filter_with_time
+                  begin
+                    filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record && filtered_time
+                    filter.measure_metrics(OneEventStream.new(time, record))
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                else
+                  begin
+                    filtered_record = filter.filter(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record
+                    filter.measure_metrics(OneEventStream.new(time, record))
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                end
+              end
+
+              new_es.add(filtered_time, filtered_record)
+            end
+          end
+          new_es
+        end
+
+        def optimizable?
+          return @optimizable unless @optimizable.nil?
+          fs_filters = filters_having_filter_stream
+          @optimizable = if fs_filters.empty?
+                           true
+                         else
+                           # skip log message when filter is only 1, because its performance is same as non optimized chain.
+                           if @filters.size > 1 && fs_filters.size >= 1
+                             $log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
+                           end
+                           false
+                         end
+        end
+
+        def filters_having_filter_stream
+          @filters_having_filter_stream ||= @filters.select do |filter|
+            filter.class.instance_methods(false).include?(:filter_stream)
+          end
+        end
+
+        def reset_optimization
+          @optimizable = nil
+          @filters_having_filter_stream = nil
+        end
+      end
+    end
+
+    def find(tag)
+      pipeline = nil
+      @match_rules.each do |rule|
+        if rule.match?(tag)
+          if rule.collector.is_a?(Plugin::Filter)
+            pipeline ||= Pipeline.new
+            pipeline.add_filter(rule.collector)
+          else
+            if pipeline
+              pipeline.set_output(rule.collector)
+            else
+              # Use Output directly when filter is not matched
+              pipeline = rule.collector
+            end
+            return pipeline
+          end
+        end
+      end
+
+      if pipeline
+        # filter is matched but no match
+        pipeline.set_output(@default_collector)
+        pipeline
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/fluent/ext_monitor_require.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# To avoid duplicated requirements, extract this logic as file.
+
+if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.7.0')
+  require 'monitor'
+else
+  begin
+    # monitor_ext is bundled since ruby 2.7.0
+    require 'ext_monitor'
+  rescue LoadError => _
+    require 'monitor'
+  end
+end

data/lib/fluent/file_wrapper.rb

@@ -0,0 +1,137 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+unless Fluent.windows?
+  Fluent::FileWrapper = File
+else
+  require 'fluent/win32api'
+
+  module Fluent
+    module FileWrapper
+      def self.open(path, mode='r')
+        io = WindowsFile.new(path, mode).io
+        if block_given?
+          v = yield io
+          io.close
+          v
+        else
+          io
+        end
+      end
+
+      def self.stat(path)
+        f = WindowsFile.new(path)
+        s = f.stat
+        f.close
+        s
+      end
+    end
+
+    class WindowsFile
+      include File::Constants
+
+      attr_reader :io
+
+      INVALID_HANDLE_VALUE = -1
+
+      def initialize(path, mode_enc='r')
+        @path = path
+        mode, enc = mode_enc.split(":", 2)
+        @io = File.open(path, mode2flags(mode))
+        @io.set_encoding(enc) if enc
+        @file_handle = Win32API._get_osfhandle(@io.to_i)
+        @io.instance_variable_set(:@file_index, self.ino)
+        def @io.ino
+          @file_index
+        end
+      end
+
+      def close
+        @io.close
+        @file_handle = INVALID_HANDLE_VALUE
+      end
+
+      # To keep backward compatibility, we continue to use GetFileInformationByHandle()
+      # to get file id.
+      # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
+      # and returned value is different with above one, former one is 64 bit while
+      # later one is 128bit.
+      def ino
+        by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
+
+        unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
+          return 0
+        end
+
+        by_handle_file_information.unpack("I11Q1")[11] # fileindex
+      end
+
+      def stat
+        raise Errno::ENOENT if delete_pending
+        s = File.stat(@path)
+        s.instance_variable_set :@ino, self.ino
+        def s.ino; @ino; end
+        s
+      end
+
+      private
+
+      def mode2flags(mode)
+        # Always inject File::Constants::SHARE_DELETE
+        # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
+        # To enable SHARE_DELETE, BINARY is also required.
+        # https://bugs.ruby-lang.org/issues/11218
+        # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
+        flags = BINARY | SHARE_DELETE
+        case mode.delete("b")
+        when "r"
+          flags |= RDONLY
+        when "r+"
+          flags |= RDWR
+        when "w"
+          flags |= WRONLY | CREAT | TRUNC
+        when "w+"
+          flags |= RDWR | CREAT | TRUNC
+        when "a"
+          flags |= WRONLY | CREAT | APPEND
+        when "a+"
+          flags |= RDWR | CREAT | APPEND
+        else
+          raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
+        end
+      end
+
+      # DeletePending is a Windows-specific file state that roughly means
+      # "this file is queued for deletion, so close any open handlers"
+      #
+      # This flag can be retrieved via GetFileInformationByHandleEx().
+      #
+      # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+      #
+      def delete_pending
+        file_standard_info = 0x01
+        bufsize = 1024
+        buf = '\0' * bufsize
+
+        unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
+          return false
+        end
+
+        return buf.unpack("QQICC")[3] != 0
+      end
+    end
+  end
+end

data/lib/fluent/filter.rb

@@ -0,0 +1,21 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/filter'
+
+module Fluent
+  Filter = Fluent::Compat::Filter
+end

data/lib/fluent/fluent_log_event_router.rb

@@ -0,0 +1,139 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/log'
+
+module Fluent
+  # DO NOT write any logic here
+  class NullFluentLogEventRouter
+    def start; end
+
+    def stop; end
+
+    def graceful_stop; end
+
+    def emit_event(_event); end
+
+    def emittable?
+      self.class != NullFluentLogEventRouter
+    end
+  end
+
+  # This class is for handling fluentd's inner log
+  # e.g. <label @FLUNT_LOG> section and <match fluent.**> section
+  class FluentLogEventRouter < NullFluentLogEventRouter
+    # @param root_agent [Fluent::RootAgent]
+    def self.build(root_agent)
+      log_event_router = nil
+
+      begin
+        log_event_agent = root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
+        log_event_router = log_event_agent.event_router
+
+        # suppress mismatched tags only for <label @FLUENT_LOG> label.
+        # it's not suppressed in default event router for non-log-event events
+        log_event_router.suppress_missing_match!
+
+        unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
+        unless unmatched_tags.empty?
+          $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags
+        end
+
+      rescue ArgumentError # ArgumentError "#{label_name} label not found"
+        # use default event router if <label @FLUENT_LOG> is missing in configuration
+        root_log_event_router = root_agent.event_router
+        event_tags = Fluent::Log.event_tags
+        if event_tags.any? { |t| root_log_event_router.match?(t) }
+          log_event_router = root_log_event_router
+
+          unmatched_tags = event_tags.select { |t| !log_event_router.match?(t) }
+          if unmatched_tags.empty?
+            $log.warn "define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead"
+          else
+            matched_sections = (event_tags - unmatched_tags).map { |tag| "<match #{tag}>" }.join(', ')
+            $log.warn "define #{matched_sections} to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead"
+            $log.warn "match for some tags of log events are not defined in top level (to be ignored)", tags: unmatched_tags
+          end
+        end
+      end
+
+      if log_event_router
+        FluentLogEventRouter.new(log_event_router)
+      else
+        $log.debug('No fluent logger for internal event')
+        NullFluentLogEventRouter.new
+      end
+    end
+
+    STOP = :stop
+    GRACEFUL_STOP = :graceful_stop
+
+    # @param event_router [Fluent::EventRouter]
+    def initialize(event_router)
+      @event_router = event_router
+      @thread = nil
+      @graceful_stop = false
+      @event_queue = Queue.new
+    end
+
+    def start
+      @thread = Thread.new do
+        $log.disable_events(Thread.current)
+
+        loop do
+          event = @event_queue.pop
+
+          case event
+          when GRACEFUL_STOP
+            @graceful_stop = true
+          when STOP
+            break
+          else
+            begin
+              tag, time, record = event
+              @event_router.emit(tag, time, record)
+            rescue => e
+              # This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
+              $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
+            end
+          end
+
+          if @graceful_stop && @event_queue.empty?
+            break
+          end
+        end
+      end
+
+      @thread.abort_on_exception = true
+    end
+
+    def stop
+      @event_queue.push(STOP)
+      # there is no problem calling Thread#join multiple times.
+      @thread && @thread.join
+    end
+
+    def graceful_stop
+      # to make sure to emit all log events into router, before shutting down
+      @event_queue.push(GRACEFUL_STOP)
+      @thread && @thread.join
+    end
+
+    def emit_event(event)
+      @event_queue.push(event)
+    end
+  end
+end

data/lib/fluent/formatter.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/formatter'
+
+module Fluent
+  Formatter = Fluent::Compat::Formatter
+  TextFormatter = Fluent::Compat::TextFormatter
+  # deprecate_constant is ruby 2.3 feature
+end

data/lib/fluent/input.rb

@@ -0,0 +1,21 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/input'
+
+module Fluent
+  Input = Fluent::Compat::Input
+end