fluentd222 1.16.2-x86_64-linux

data/lib/fluent/file_wrapper.rb

@@ -0,0 +1,137 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+unless Fluent.windows?
+  Fluent::FileWrapper = File
+else
+  require 'fluent/win32api'
+
+  module Fluent
+    module FileWrapper
+      def self.open(path, mode='r')
+        io = WindowsFile.new(path, mode).io
+        if block_given?
+          v = yield io
+          io.close
+          v
+        else
+          io
+        end
+      end
+
+      def self.stat(path)
+        f = WindowsFile.new(path)
+        s = f.stat
+        f.close
+        s
+      end
+    end
+
+    class WindowsFile
+      include File::Constants
+
+      attr_reader :io
+
+      INVALID_HANDLE_VALUE = -1
+
+      def initialize(path, mode_enc='r')
+        @path = path
+        mode, enc = mode_enc.split(":", 2)
+        @io = File.open(path, mode2flags(mode))
+        @io.set_encoding(enc) if enc
+        @file_handle = Win32API._get_osfhandle(@io.to_i)
+        @io.instance_variable_set(:@file_index, self.ino)
+        def @io.ino
+          @file_index
+        end
+      end
+
+      def close
+        @io.close
+        @file_handle = INVALID_HANDLE_VALUE
+      end
+
+      # To keep backward compatibility, we continue to use GetFileInformationByHandle()
+      # to get file id.
+      # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
+      # and returned value is different with above one, former one is 64 bit while
+      # later one is 128bit.
+      def ino
+        by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
+
+        unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
+          return 0
+        end
+
+        by_handle_file_information.unpack("I11Q1")[11] # fileindex
+      end
+
+      def stat
+        raise Errno::ENOENT if delete_pending
+        s = File.stat(@path)
+        s.instance_variable_set :@ino, self.ino
+        def s.ino; @ino; end
+        s
+      end
+
+      private
+
+      def mode2flags(mode)
+        # Always inject File::Constants::SHARE_DELETE
+        # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
+        # To enable SHARE_DELETE, BINARY is also required.
+        # https://bugs.ruby-lang.org/issues/11218
+        # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
+        flags = BINARY | SHARE_DELETE
+        case mode.delete("b")
+        when "r"
+          flags |= RDONLY
+        when "r+"
+          flags |= RDWR
+        when "w"
+          flags |= WRONLY | CREAT | TRUNC
+        when "w+"
+          flags |= RDWR | CREAT | TRUNC
+        when "a"
+          flags |= WRONLY | CREAT | APPEND
+        when "a+"
+          flags |= RDWR | CREAT | APPEND
+        else
+          raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
+        end
+      end
+
+      # DeletePending is a Windows-specific file state that roughly means
+      # "this file is queued for deletion, so close any open handlers"
+      #
+      # This flag can be retrieved via GetFileInformationByHandleEx().
+      #
+      # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+      #
+      def delete_pending
+        file_standard_info = 0x01
+        bufsize = 1024
+        buf = '\0' * bufsize
+
+        unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
+          return false
+        end
+
+        return buf.unpack("QQICC")[3] != 0
+      end
+    end
+  end
+end

data/lib/fluent/filter.rb

@@ -0,0 +1,21 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/filter'
+
+module Fluent
+  Filter = Fluent::Compat::Filter
+end

data/lib/fluent/fluent_log_event_router.rb

@@ -0,0 +1,139 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/log'
+
+module Fluent
+  # DO NOT write any logic here
+  class NullFluentLogEventRouter
+    def start; end
+
+    def stop; end
+
+    def graceful_stop; end
+
+    def emit_event(_event); end
+
+    def emittable?
+      self.class != NullFluentLogEventRouter
+    end
+  end
+
+  # This class is for handling fluentd's inner log
+  # e.g. <label @FLUNT_LOG> section and <match fluent.**> section
+  class FluentLogEventRouter < NullFluentLogEventRouter
+    # @param root_agent [Fluent::RootAgent]
+    def self.build(root_agent)
+      log_event_router = nil
+
+      begin
+        log_event_agent = root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
+        log_event_router = log_event_agent.event_router
+
+        # suppress mismatched tags only for <label @FLUENT_LOG> label.
+        # it's not suppressed in default event router for non-log-event events
+        log_event_router.suppress_missing_match!
+
+        unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
+        unless unmatched_tags.empty?
+          $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags
+        end
+
+      rescue ArgumentError # ArgumentError "#{label_name} label not found"
+        # use default event router if <label @FLUENT_LOG> is missing in configuration
+        root_log_event_router = root_agent.event_router
+        event_tags = Fluent::Log.event_tags
+        if event_tags.any? { |t| root_log_event_router.match?(t) }
+          log_event_router = root_log_event_router
+
+          unmatched_tags = event_tags.select { |t| !log_event_router.match?(t) }
+          if unmatched_tags.empty?
+            $log.warn "define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead"
+          else
+            matched_sections = (event_tags - unmatched_tags).map { |tag| "<match #{tag}>" }.join(', ')
+            $log.warn "define #{matched_sections} to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead"
+            $log.warn "match for some tags of log events are not defined in top level (to be ignored)", tags: unmatched_tags
+          end
+        end
+      end
+
+      if log_event_router
+        FluentLogEventRouter.new(log_event_router)
+      else
+        $log.debug('No fluent logger for internal event')
+        NullFluentLogEventRouter.new
+      end
+    end
+
+    STOP = :stop
+    GRACEFUL_STOP = :graceful_stop
+
+    # @param event_router [Fluent::EventRouter]
+    def initialize(event_router)
+      @event_router = event_router
+      @thread = nil
+      @graceful_stop = false
+      @event_queue = Queue.new
+    end
+
+    def start
+      @thread = Thread.new do
+        $log.disable_events(Thread.current)
+
+        loop do
+          event = @event_queue.pop
+
+          case event
+          when GRACEFUL_STOP
+            @graceful_stop = true
+          when STOP
+            break
+          else
+            begin
+              tag, time, record = event
+              @event_router.emit(tag, time, record)
+            rescue => e
+              # This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
+              $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
+            end
+          end
+
+          if @graceful_stop && @event_queue.empty?
+            break
+          end
+        end
+      end
+
+      @thread.abort_on_exception = true
+    end
+
+    def stop
+      @event_queue.push(STOP)
+      # there is no problem calling Thread#join multiple times.
+      @thread && @thread.join
+    end
+
+    def graceful_stop
+      # to make sure to emit all log events into router, before shutting down
+      @event_queue.push(GRACEFUL_STOP)
+      @thread && @thread.join
+    end
+
+    def emit_event(event)
+      @event_queue.push(event)
+    end
+  end
+end

data/lib/fluent/formatter.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/formatter'
+
+module Fluent
+  Formatter = Fluent::Compat::Formatter
+  TextFormatter = Fluent::Compat::TextFormatter
+  # deprecate_constant is ruby 2.3 feature
+end

data/lib/fluent/input.rb

@@ -0,0 +1,21 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/input'
+
+module Fluent
+  Input = Fluent::Compat::Input
+end

data/lib/fluent/label.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/agent'
+
+module Fluent
+  class Label < Agent
+    def initialize(name, log:)
+      super(log: log)
+
+      @context = name
+      @root_agent = nil
+    end
+
+    attr_accessor :root_agent
+
+    def configure(conf)
+      super
+
+      if conf.elements('match').size == 0
+        raise ConfigError, "Missing <match> sections in <label #{@context}> section"
+      end
+    end
+
+    def emit_error_event(tag, time, record, e)
+      @root_agent.emit_error_event(tag, time, record, e)
+    end
+
+    def handle_emits_error(tag, es, e)
+      @root_agent.handle_emits_error(tag, es, e)
+    end
+  end
+end

data/lib/fluent/load.rb

@@ -0,0 +1,34 @@
+require 'socket'
+require 'fcntl'
+require 'time'
+require 'stringio'
+require 'fileutils'
+require 'json'
+require 'yajl'
+require 'uri'
+require 'msgpack'
+require 'strptime'
+begin
+  require 'sigdump/setup'
+rescue
+  # ignore setup error on Win or similar platform which doesn't support signal
+end
+require 'cool.io'
+
+require 'fluent/time'
+require 'fluent/env'
+require 'fluent/version'
+require 'fluent/log'
+require 'fluent/config'
+require 'fluent/engine'
+require 'fluent/rpc'
+require 'fluent/mixin'
+require 'fluent/plugin'
+require 'fluent/parser'
+require 'fluent/formatter'
+require 'fluent/event'
+require 'fluent/input'
+require 'fluent/output'
+require 'fluent/filter'
+require 'fluent/match'
+require 'fluent/ext_monitor_require'

data/lib/fluent/log/console_adapter.rb

@@ -0,0 +1,66 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'console/terminal/logger'
+
+module Fluent
+  class Log
+    # Async gem which is used by http_server helper switched logger mechanism to
+    # Console gem which isn't complatible with Ruby's standard Logger (since
+    # v1.17). This class adapts it to Fluentd's logger mechanism.
+    class ConsoleAdapter < Console::Terminal::Logger
+      def self.wrap(logger)
+        _, level = Console::Logger::LEVELS.find { |key, value|
+          if logger.level <= 0
+            key == :debug
+          else
+            value == logger.level - 1
+          end
+        }
+        Console::Logger.new(ConsoleAdapter.new(logger), level: level)
+      end
+
+      def initialize(logger)
+        @logger = logger
+        # When `verbose` is `true`, following items will be added as a prefix or
+        # suffix of the subject:
+        #   * Severity
+        #   * Object ID
+        #   * PID
+        #   * Time
+        # Severity and Time are added by Fluentd::Log too so they are redundant.
+        # PID is the worker's PID so it's also redundant.
+        # Object ID will be too verbose in usual cases.
+        # So set it as `false` here to suppress redundant items.
+        super(StringIO.new, verbose: false)
+      end
+
+      def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)
+        if LEVEL_TEXT.include?(severity.to_s)
+          level = severity
+        else
+          @logger.warn("Unknown severity: #{severity}")
+          level = 'warn'
+        end
+
+        @io.seek(0)
+        @io.truncate(0)
+        super
+        @logger.send(level, @io.string.chomp)
+      end
+    end
+  end
+end