fluentd 1.19.3 → 1.20.0

data/lib/fluent/plugin/in_tail/stat_watcher.rb

@@ -0,0 +1,52 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'cool.io'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    class StatWatcher < Coolio::StatWatcher
+      def initialize(path, log, &callback)
+        @callback = callback
+        @log = log
+        super(path)
+      end
+
+      def on_change(prev, cur)
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+
+    class TimerTrigger < Coolio::TimerWatcher
+      def initialize(interval, log, repeat: true, &callback)
+        @log = log
+        @callback = callback
+        super(interval, repeat)
+      end
+
+      def on_timer
+        @callback.call
+      rescue => e
+        @log.error e.to_s
+        @log.error_backtrace
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_tail/tail_watcher.rb

@@ -0,0 +1,276 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'fluent/file_wrapper'
+require 'fluent/plugin/in_tail/position_file'
+require 'fluent/plugin/in_tail/compatibility'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    class TailWatcher
+      def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, file_feed, io_handler_build, metrics)
+        @path = target_info.path
+        @ino = target_info.ino
+        @pe = pe || MemoryPositionEntry.new
+        @read_from_head = read_from_head
+        @follow_inodes = follow_inodes
+        @update_watcher = update_watcher
+        @log = log
+        @rotate_handler = RotateHandler.new(log, &method(:on_rotate))
+        @file_feed = file_feed
+        @io_handler = nil
+        @io_handler_build = io_handler_build
+        @metrics = metrics
+        @watchers = []
+        @detached = false
+        @close_pending = false
+      end
+
+      attr_reader :path, :ino
+
+      def detached?
+        @detached
+      end
+      attr_reader :pe
+      attr_reader :file_feed
+      attr_accessor :unwatched  # This is used for removing position entry from PositionFile
+      attr_reader :watchers
+      attr_accessor :group_watcher
+
+      def tag
+        @parsed_tag ||= @path.tr('/', '.').squeeze('.').gsub(/^\./, '')
+      end
+
+      def register_watcher(watcher)
+        @watchers << watcher
+      end
+
+      def detach(shutdown_start_time = nil)
+        @detached = true
+        if @io_handler
+          ready_to_shutdown(shutdown_start_time)
+          @io_handler.on_notify
+        end
+        @file_feed&.close(self)
+      end
+
+      def ready_to_shutdown(shutdown_start_time)
+        @shutdown_start_time = shutdown_start_time
+        @io_handler.ready_to_shutdown(shutdown_start_time) if @io_handler
+      end
+
+      def close
+        if @io_handler&.pending? || @close_pending
+          @close_pending = true
+          return
+        end
+
+        force_close
+      end
+
+      def force_close
+        @close_pending = false
+        if @io_handler
+          @io_handler.close
+          @io_handler = nil
+        end
+      end
+
+      def defer_close
+        @close_pending = true
+      end
+
+      def close_if_ready(force: false)
+        return false unless @close_pending && @io_handler && !@io_handler.pending?
+        return false unless force || @io_handler.drained?
+
+        force_close
+        true
+      end
+
+      def eof?
+        @io_handler.nil? || @io_handler.eof?
+      end
+
+      def drained?
+        @io_handler.nil? || @io_handler.drained?
+      end
+
+      def pending?
+        @io_handler&.pending? || false
+      end
+
+      def on_notify
+        if @io_handler&.pending?
+          @io_handler.on_notify
+          return
+        end
+
+        begin
+          stat = Fluent::FileWrapper.stat(@path)
+        rescue Errno::ENOENT, Errno::EACCES
+          # moved or deleted
+          stat = nil
+        end
+
+        @rotate_handler.on_notify(stat) if @rotate_handler
+        read_more
+      end
+
+      def complete_async(success, resume: true)
+        return false unless @io_handler
+
+        should_notify = @io_handler.complete_async(success)
+        if @detached
+          read_more if resume && (should_notify || success)
+          close_if_ready
+        else
+          on_notify if resume && should_notify
+        end
+        should_notify
+      end
+
+      def read_more
+        @file_feed&.on_notify(self)
+        @io_handler.on_notify if @io_handler
+      end
+
+      def on_rotate(stat)
+        if @io_handler.nil?
+          if stat
+            # first time
+            fsize = stat.size
+            inode = stat.ino
+
+            last_inode = @pe.read_inode
+            if inode == last_inode
+              # rotated file has the same inode number with the last file.
+              # assuming following situation:
+              #   a) file was once renamed and backed, or
+              #   b) symlink or hardlink to the same file is recreated
+              # in either case of a and b, seek to the saved position
+              #   c) file was once renamed, truncated and then backed
+              # in this case, consider it truncated
+              @pe.update(inode, 0) if fsize < @pe.read_pos
+            elsif last_inode != 0
+              # this is FilePositionEntry and fluentd once started.
+              # read data from the head of the rotated file.
+              # logs never duplicate because this file is a rotated new file.
+              @pe.update(inode, 0)
+            else
+              # this is MemoryPositionEntry or this is the first time fluentd started.
+              # seek to the end of the any files.
+              # logs may duplicate without this seek because it's not sure the file is
+              # existent file or rotated new file.
+              pos = @read_from_head ? 0 : fsize
+              @pe.update(inode, pos)
+            end
+            @io_handler = io_handler
+          else
+            @io_handler = NullIOHandler.new
+          end
+        else
+          watcher_needs_update = false
+
+          if stat
+            inode = stat.ino
+            if inode == @pe.read_inode # truncated
+              @pe.update_pos(0)
+              @io_handler.close
+            elsif !@io_handler.opened? # There is no previous file. Reuse TailWatcher
+              @pe.update(inode, 0)
+            else # file is rotated and new file found
+              watcher_needs_update = true
+              # Handle the old log file before renewing TailWatcher [fluentd#1055]
+              @io_handler.on_notify
+            end
+          else # file is rotated and new file not found
+            # Clear RotateHandler to avoid duplicated file watch in same path.
+            @rotate_handler = nil
+            watcher_needs_update = true
+          end
+
+          if watcher_needs_update
+            if @follow_inodes
+              # If stat is nil (file not present), NEED to stop and discard this watcher.
+              #   When the file is disappeared but is resurrected soon, then `#refresh_watcher`
+              #   can't recognize this TailWatcher needs to be stopped.
+              #   This can happens when the file is rotated.
+              #   If a notify comes before the new file for the path is created during rotation,
+              #   then it appears as if the file was resurrected once it disappeared.
+              # Don't want to swap state because we need latest read offset in pos file even after rotate_wait
+              @update_watcher.call(self, @pe, stat&.ino)
+            else
+              # Permit to handle if stat is nil (file not present).
+              # If a file is mv-ed and a new file is created during
+              # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
+              # and `#stop_watchers()` for the path because `target_paths_hash`
+              # always contains the path.
+              @update_watcher.call(self, swap_state(@pe), stat&.ino)
+            end
+          else
+            @log.info "detected rotation of #{@path}"
+            @io_handler = io_handler
+          end
+          @metrics.rotated.inc
+        end
+      end
+
+      def io_handler
+        handler = @io_handler_build.call(self, @path)
+        handler.ready_to_shutdown(@shutdown_start_time) if @shutdown_start_time
+        handler
+      end
+
+      def swap_state(pe)
+        # Use MemoryPositionEntry for rotated file temporary
+        mpe = MemoryPositionEntry.new
+        mpe.update(pe.read_inode, pe.read_pos)
+        @pe = mpe
+        pe # This pe will be updated in on_rotate after TailWatcher is initialized
+      end
+
+      class RotateHandler
+        def initialize(log, &on_rotate)
+          @log = log
+          @inode = nil
+          @fsize = -1  # first
+          @on_rotate = on_rotate
+        end
+
+        def on_notify(stat)
+          if stat.nil?
+            inode = nil
+            fsize = 0
+          else
+            inode = stat.ino
+            fsize = stat.size
+          end
+
+          if @inode != inode || fsize < @fsize
+            @on_rotate.call(stat)
+          end
+          @inode = inode
+          @fsize = fsize
+        rescue
+          @log.error $!.to_s
+          @log.error_backtrace
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_tail/worker_pool.rb

@@ -0,0 +1,298 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'socket'
+require 'fluent/env'
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    class WorkerPool
+      class WorkerStopped < StandardError; end
+      Task = Struct.new(:key, :bytes, :payload)
+      Result = Struct.new(:task, :value, :error)
+
+      def initialize(num_threads:, task_limit:, byte_limit:, thread_create:, notify:, &process)
+        unless num_threads > 0 && task_limit > 0 && byte_limit > 0
+          raise ArgumentError, 'worker and queue limits must be positive'
+        end
+        raise ArgumentError, 'processor is required' unless process
+
+        @num_threads = num_threads
+        @task_limit = task_limit
+        @byte_limit = byte_limit
+        @thread_create = thread_create
+        @notify = notify
+        @process = process
+        @mutex = Mutex.new
+        @jobs = Queue.new
+        @results = Queue.new
+        @active = {}
+        @bytes = 0
+        @regular_bytes = 0
+        @oversized_task = nil
+        @threads = []
+        @state = :new
+        @failure = nil
+      end
+
+      def start
+        @mutex.synchronize do
+          raise 'worker pool already started' unless @state == :new
+          @state = :starting
+        end
+        begin
+          @num_threads.times do |index|
+            @threads << @thread_create.call(:"in_tail_worker_#{index}") { run(index) }
+          end
+          @mutex.synchronize do
+            raise(@failure || WorkerStopped.new('pool stopped during start')) unless @state == :starting
+            @state = :running
+          end
+          self
+        rescue
+          stop
+          join
+          raise
+        end
+      end
+
+      # Capacity remains reserved until emit and position handling acknowledge
+      # the result, including while a parsed batch waits in the result queue.
+      def submit(key, payload, bytes:)
+        raise ArgumentError, 'batch bytes must be nonnegative' if bytes < 0
+
+        @mutex.synchronize do
+          return false unless @state == :running
+          return false if @active.key?(key) || @active.size >= @task_limit
+          # A single over-limit batch may use an extra reservation alongside the
+          # bounded regular budget so it does not force parsing on the event loop.
+          if bytes > @byte_limit
+            return false if @oversized_task
+          else
+            return false if @regular_bytes + bytes > @byte_limit
+          end
+
+          task = Task.new(key, bytes, payload).freeze
+          @active[key] = task
+          @bytes += bytes
+          if bytes > @byte_limit
+            @oversized_task = task
+          else
+            @regular_bytes += bytes
+          end
+          @jobs << task
+          true
+        end
+      end
+
+      def next_result
+        @results.pop(true)
+      rescue ThreadError
+        nil
+      end
+
+      def acknowledge(result)
+        @mutex.synchronize do
+          task = result.task
+          unless @active[task.key].equal?(task)
+            raise ArgumentError, 'result is no longer active'
+          end
+
+          @active.delete(task.key)
+          @bytes -= task.bytes
+          if task.bytes > @byte_limit
+            @oversized_task = nil
+          else
+            @regular_bytes -= task.bytes
+          end
+        end
+      end
+
+      def statistics
+        @mutex.synchronize { { tasks: @active.size, bytes: @bytes, state: @state }.freeze }
+      end
+
+      def failure
+        @mutex.synchronize { @failure }
+      end
+
+      def stop
+        @mutex.synchronize do
+          @state = :stopping unless @state == :failed
+          @jobs.close
+        end
+      end
+
+      # The source joins after draining results; the event loop never joins.
+      def join(timeout: nil)
+        deadline = timeout && Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+        @threads.each do |thread|
+          remaining = deadline && [deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max
+          return false unless thread.join(remaining)
+        end
+        true
+      end
+
+      def kill
+        @threads.each(&:kill)
+        @threads.each(&:join)
+      end
+
+      private
+
+      def run(index)
+        normal_exit = false
+        while (task = @jobs.pop)
+          value = nil
+          error = nil
+          completed = false
+          begin
+            value = @process.call(index, task.payload)
+            completed = true
+          rescue Exception => e
+            # Deliver task failures to the owner instead of losing an active
+            # file or terminating a helper thread with abort_on_exception.
+            error = e
+            completed = true
+          ensure
+            error = WorkerStopped.new('worker stopped during a task') unless completed
+            publish(Result.new(task, value, error).freeze)
+          end
+        end
+        normal_exit = true
+      ensure
+        fail_pool(WorkerStopped.new('worker exited unexpectedly')) unless normal_exit
+      end
+
+      def publish(result)
+        @results << result
+        begin
+          @notify.call
+        rescue Exception => error
+          fail_pool(error)
+        end
+      end
+
+      def fail_pool(error)
+        @mutex.synchronize do
+          @failure ||= error
+          @state = :failed
+          @jobs.close
+        end
+        begin
+          while (task = @jobs.pop(true))
+            @results << Result.new(task, nil, error).freeze
+          end
+        rescue ThreadError
+          # All queued jobs have been accounted for.
+        end
+        begin
+          @notify.call
+        rescue Exception => notification_error
+          @mutex.synchronize { @failure ||= notification_error }
+        end
+      end
+    end
+
+    # Captures the built-in hook implementations once and checks whether a
+    # TailInput instance resolves each hook to the same method body. A mismatch
+    # means worker execution must fall back to the synchronous compatibility
+    # path; this deliberately does not try to infer safety from method owners.
+    class WorkerCompatibility
+      HOOKS = %i[
+        receive_lines flush_buffer convert_line_to_event parse_singleline
+        parse_multilines setup_watcher io_handler build_line_feeder
+      ].freeze
+
+      def initialize(base_class)
+        @hooks = HOOKS.to_h { |name| [name, base_class.instance_method(name)] }.freeze
+      end
+
+      def compatible?(plugin)
+        incompatible_hooks(plugin).empty?
+      end
+
+      def incompatible_hooks(plugin)
+        @hooks.filter_map do |name, built_in|
+          name unless plugin.singleton_class.instance_method(name) == built_in
+        rescue NameError
+          name
+        end
+      end
+    end
+
+    class WorkerCompletionWatcher < Coolio::IOWatcher
+      def initialize(log: nil, &on_completion)
+        @log = log
+        if Fluent.windows?
+          # Winsock select cannot monitor IO.pipe handles. Use connected
+          # loopback sockets, as libev does for its own Windows wakeups.
+          begin
+            listener = TCPServer.new('127.0.0.1', 0)
+            @writer = TCPSocket.new('127.0.0.1', listener.addr[1])
+            @reader = listener.accept
+          rescue
+            @reader&.close
+            @writer&.close
+            raise
+          ensure
+            listener&.close
+          end
+        else
+          @reader, @writer = IO.pipe
+        end
+        super(@reader)
+        @on_completion = on_completion
+      end
+
+      def signal
+        @writer.write_nonblock("\0")
+        true
+      rescue IO::WaitWritable
+        # A full pipe already has a notification pending on the event loop.
+        false
+      end
+
+      def on_signal
+        @on_completion.call
+      end
+
+      def on_readable
+        notified = false
+        begin
+          loop do
+            @reader.read_nonblock(4096)
+            notified = true
+          end
+        rescue IO::WaitReadable
+          return unless notified
+        end
+        on_signal
+      rescue => error
+        @log&.error 'unexpected error while processing in_tail worker completions', error: error
+        @log&.error_backtrace
+      end
+
+      def close
+        detach if attached?
+        @reader.close unless @reader.closed?
+        @writer.close unless @writer.closed?
+      end
+    end
+  end
+end