fluentd 1.19.3 → 1.20.0

data/lib/fluent/plugin/in_tail/group_watch.rb

@@ -78,19 +78,13 @@ module Fluent::Plugin
         end
       end
 
-      def add_path_to_group_watcher(path)
+      def add_path_to_group_watcher(path, watcher)
         return nil if @group.nil?
         group_watcher = find_group_from_metadata(path)
-        group_watcher.add(path) unless group_watcher.include?(path)
+        group_watcher.add(path, watcher)
         group_watcher
       end
 
-      def remove_path_from_group_watcher(path)
-        return if @group.nil?
-        group_watcher = find_group_from_metadata(path)
-        group_watcher.delete(path)
-      end
-
       def construct_group_key(named_captures)
         match_rule = []
         @group_keys.each { |key|
@@ -135,6 +129,7 @@ module Fluent::Plugin
       FileCounter = Struct.new(
         :number_lines_read,
         :start_reading_time,
+        :watchers,
       )
 
       def initialize(rate_period = 60, limit = -1)
@@ -143,8 +138,8 @@ module Fluent::Plugin
         @limit = limit
       end
 
-      def add(path)
-        @current_paths[path] = FileCounter.new(0, nil)
+      def add(path, watcher)
+        (@current_paths[path] ||= FileCounter.new(0, nil, Set.new)).watchers << watcher
       end
 
       def include?(path)
@@ -155,8 +150,11 @@ module Fluent::Plugin
         @current_paths.size
       end
 
-      def delete(path)
-        @current_paths.delete(path)
+      def delete(path, watcher)
+        counter = @current_paths[path]
+        return unless counter
+        counter.watchers.delete(watcher)
+        @current_paths.delete(path) if counter.watchers.empty?
       end
 
       def update_reading_time(path)
@@ -185,7 +183,9 @@ module Fluent::Plugin
         return true if @limit == 0
 
         return false if @limit < 0
-        return false if @current_paths[path].number_lines_read < @limit / size
+        # Give every path at least one line. With more paths than the limit, the integer division
+        # leaves no allotment, and a path that has not started reading yet has no start time to compare.
+        return false if @current_paths[path].number_lines_read < [@limit / size, 1].max
 
         # update_reading_time(path)
         if limit_time_period_reached?(path) # Exceeds limit

data/lib/fluent/plugin/in_tail/io_handler.rb

@@ -0,0 +1,391 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'fluent/clock'
+require 'fluent/file_wrapper'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    class TailWatcher
+      class FIFO
+        def initialize(encoding, log, max_line_size=nil, encoding_to_convert=nil)
+          @buffer = ''.force_encoding(encoding)
+          @eol = "\n".encode(encoding).freeze
+          @encoding_to_convert = encoding_to_convert
+          @max_line_size = max_line_size
+          @skip_current_line = false
+          @skipping_current_line_bytesize = 0
+          @log = log
+        end
+
+        attr_reader :buffer, :max_line_size
+
+        def <<(chunk)
+          @buffer << chunk
+        end
+
+        def convert(s)
+          if @encoding_to_convert
+            s.encode!(@encoding_to_convert)
+          else
+            s
+          end
+        rescue
+          s.encode!(@encoding_to_convert, :invalid => :replace, :undef => :replace)
+        end
+
+        def read_lines(lines)
+          idx = @buffer.index(@eol)
+          has_skipped_line = false
+
+          until idx.nil?
+            # Using freeze and slice is faster than slice!
+            # See https://github.com/fluent/fluentd/pull/2527
+            @buffer.freeze
+            slice_position = idx + 1
+            rbuf = @buffer.slice(0, slice_position)
+            @buffer = @buffer.slice(slice_position, @buffer.size - slice_position)
+            idx = @buffer.index(@eol)
+
+            is_long_line = @max_line_size && (
+              @skip_current_line || rbuf.bytesize > @max_line_size
+            )
+
+            if is_long_line
+              @log.warn "received line length is longer than #{@max_line_size}"
+              if @skip_current_line
+                @log.debug("The continuing line is finished. Finally discarded data: ") { convert(rbuf).chomp }
+              else
+                @log.debug("skipped line: ") { convert(rbuf).chomp }
+              end
+              has_skipped_line = true
+              @skip_current_line = false
+              @skipping_current_line_bytesize = 0
+              next
+            end
+
+            lines << convert(rbuf)
+          end
+
+          is_long_current_line = @max_line_size && (
+            @skip_current_line || @buffer.bytesize > @max_line_size
+          )
+
+          if is_long_current_line
+            @log.debug(
+              "The continuing current line length is longer than #{@max_line_size}." +
+              " The received data will be discarded until this line is finished." +
+              " Discarded data: "
+            ) { convert(@buffer).chomp }
+            @skip_current_line = true
+            @skipping_current_line_bytesize += @buffer.bytesize
+            @buffer.clear
+          end
+
+          return has_skipped_line
+        end
+
+        def reading_bytesize
+          return @skipping_current_line_bytesize if @skip_current_line
+          @buffer.bytesize
+        end
+      end
+
+      class IOHandler
+        BYTES_TO_READ = 64 * 1024
+        SHUTDOWN_TIMEOUT = 5
+        ASYNC_PENDING = :async_pending
+
+        attr_accessor :shutdown_timeout
+
+        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, max_line_size: nil, log:, open_on_every_update:, encoding: Encoding::ASCII_8BIT, encoding_to_convert: nil, metrics:, &receive_lines)
+          @watcher = watcher
+          @path = path
+          @read_lines_limit = read_lines_limit
+          @read_bytes_limit_per_second = read_bytes_limit_per_second
+          @receive_lines = receive_lines
+          @open_on_every_update = open_on_every_update
+          @encoding = encoding
+          @fifo = FIFO.new(encoding, log, max_line_size, encoding_to_convert)
+          @lines = []
+          @io = nil
+          @notify_mutex = Mutex.new
+          @log = log
+          @start_reading_time = nil
+          @number_bytes_read = 0
+          @shutdown_start_time = nil
+          @shutdown_timeout = SHUTDOWN_TIMEOUT
+          @shutdown_mutex = Mutex.new
+          @eof = false
+          @metrics = metrics
+          @async_pending = false
+          @notify_pending = false
+          @pending_position = nil
+          @pending_read_more = false
+
+          @log.info "following tail of #{@path}"
+        end
+
+        def group_watcher
+          @watcher.group_watcher
+        end
+
+        def on_notify
+          @notify_mutex.synchronize { handle_notify }
+        end
+
+        def ready_to_shutdown(shutdown_start_time = nil)
+          @shutdown_mutex.synchronize {
+            @shutdown_start_time =
+              shutdown_start_time || Fluent::Clock.now
+          }
+        end
+
+        def close
+          if @io && !@io.closed?
+            @io.close
+            @io = nil
+            @metrics.closed.inc
+          end
+        end
+
+        def opened?
+          !!@io
+        end
+
+        def eof?
+          @eof
+        end
+
+        def drained?
+          @eof && @lines.empty? && !@async_pending
+        end
+
+        def pending?
+          @async_pending
+        end
+
+        # Completes a receive callback that returned ASYNC_PENDING. The caller
+        # must invoke TailWatcher#on_notify when this returns true.
+        def complete_async(success)
+          @notify_mutex.synchronize do
+            raise 'no asynchronous receive is pending' unless @async_pending
+
+            begin
+              if success
+                @watcher.pe.update_pos(@pending_position)
+                @lines.clear
+              end
+              should_notify = (success && @pending_read_more) || @notify_pending
+              @pending_read_more = false if success
+              should_notify
+            rescue => error
+              @log.error error.to_s
+              @log.error_backtrace
+              @notify_pending
+            ensure
+              @async_pending = false
+              @notify_pending = false
+              @pending_position = nil
+            end
+          end
+        end
+
+        private
+
+        def limit_bytes_per_second_reached?
+          return false if @read_bytes_limit_per_second < 0 # not enabled by conf
+          return false if @number_bytes_read < @read_bytes_limit_per_second
+
+          @start_reading_time ||= Fluent::Clock.now
+          time_spent_reading = Fluent::Clock.now - @start_reading_time
+          @log.debug("time_spent_reading: #{time_spent_reading} #{ @watcher.path}")
+
+          if time_spent_reading < 1
+            true
+          else
+            @start_reading_time = nil
+            @number_bytes_read = 0
+            false
+          end
+        end
+
+        def should_shutdown_now?
+          # Ensure to read all remaining lines, but abort immediately if it
+          # seems to take too long time.
+          @shutdown_mutex.synchronize {
+            return false if @shutdown_start_time.nil?
+            return Fluent::Clock.now - @shutdown_start_time > @shutdown_timeout
+          }
+        end
+
+        def handle_notify
+          if @async_pending
+            @notify_pending = true
+            return
+          end
+
+          # A notification may arrive while a rejected batch is waiting for capacity.
+          @pending_read_more = true unless @lines.empty?
+
+          if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path)
+            @metrics.throttled.inc
+            return
+          end
+
+          with_io do |io|
+            iobuf = ''.force_encoding(@encoding)
+            begin
+              read_more = false
+              has_skipped_line = false
+
+              if !io.nil? && @lines.empty?
+                begin
+                  while true
+                    @start_reading_time ||= Fluent::Clock.now
+                    group_watcher&.update_reading_time(@path)
+
+                    data = io.readpartial(BYTES_TO_READ, iobuf)
+                    @eof = false
+                    @number_bytes_read += data.bytesize
+                    @fifo << data
+
+                    n_lines_before_read = @lines.size
+                    has_skipped_line = @fifo.read_lines(@lines) || has_skipped_line
+                    group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
+
+                    group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
+                    @log.debug "Reading Limit exceeded #{@path} #{group_watcher.number_lines_read}" if group_watcher_limit
+
+                    if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
+                      # Just get out from tailing loop.
+                      @metrics.throttled.inc if group_watcher_limit || limit_bytes_per_second_reached?
+                      read_more = false
+                      break
+                    end
+
+                    if @lines.size >= @read_lines_limit
+                      # not to use too much memory in case the file is very large
+                      read_more = true
+                      break
+                    end
+                  end
+                rescue EOFError
+                  @eof = true
+                ensure
+                  iobuf.clear
+                end
+              end
+
+              if @lines.empty?
+                @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize) if has_skipped_line
+              else
+                received = @receive_lines.call(@lines, @watcher)
+                if received == ASYNC_PENDING
+                  @pending_position = io.pos - @fifo.reading_bytesize
+                  @pending_read_more ||= read_more
+                  @async_pending = true
+                  read_more = false
+                elsif received
+                  @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize)
+                  @lines.clear
+                  @pending_read_more = false
+                else
+                  # A rejected worker submission must retain the continuation for retry.
+                  @pending_read_more ||= read_more
+                  read_more = false
+                end
+              end
+            end while read_more
+          end
+        end
+
+        def open
+          io = Fluent::FileWrapper.open(@path)
+          io.seek(@watcher.pe.read_pos + @fifo.reading_bytesize)
+          @metrics.opened.inc
+          io
+        rescue RangeError
+          io.close if io
+          raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.reading_bytesize.to_s(16)}"
+        rescue Errno::EACCES => e
+          @log.warn "#{e}"
+          nil
+        rescue Errno::ENOENT
+          nil
+        end
+
+        def with_io
+          if @open_on_every_update
+            io = open
+            begin
+              yield io
+            ensure
+              io.close unless io.nil?
+            end
+          else
+            @io ||= open
+            yield @io
+            @eof = true if @io.nil?
+          end
+        rescue WatcherSetupError => e
+          close
+          @eof = true
+          raise e
+        rescue
+          @log.error $!.to_s
+          @log.error_backtrace
+          close
+          @eof = true
+        end
+      end
+
+      class NullIOHandler
+        def initialize
+        end
+
+        def ready_to_shutdown(shutdown_start_time = nil)
+        end
+
+        def io
+        end
+
+        def on_notify
+        end
+
+        def close
+        end
+
+        def opened?
+          false
+        end
+
+        def eof?
+          true
+        end
+
+        def drained?
+          true
+        end
+
+        def pending?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_tail/line_feeder.rb

@@ -0,0 +1,230 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'fluent/event'
+require 'fluent/plugin/buffer'
+require 'fluent/clock'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    # LineFeeder provides the built-in line parsing and emitting for TailInput.
+    # To preserve compatibility with TailInput subclasses, TailInput passes its
+    # line-processing methods as handlers. This keeps subclass overrides in the
+    # call path, and super reaches this built-in implementation.
+    class LineFeeder
+      def initialize(parser:, router_provider:, log:, tag:, tag_prefix:, tag_suffix:, path_key:, emit_unmatched_lines:, multiline_mode:, parse_handler: nil, convert_handler: nil, flush_handler: nil)
+        @parser = parser
+        @router_provider = router_provider
+        @log = log
+        @tag = tag
+        @tag_prefix = tag_prefix
+        @tag_suffix = tag_suffix
+        @path_key = path_key
+        @emit_unmatched_lines = emit_unmatched_lines
+        @multiline_mode = multiline_mode
+        @parse_handler = parse_handler || if multiline_mode
+                                           method(:parse_multilines)
+                                         else
+                                           method(:parse_singleline)
+                                         end
+        @convert_handler = convert_handler || method(:convert_line_to_event)
+        @flush_handler = flush_handler || method(:flush_buffer)
+      end
+
+      # Builds a per-file feed. The flush interval applies only to multiline
+      # parsers with a firstline.
+      def new_file_feed(flush_interval: nil)
+        timer_flush_interval = (@multiline_mode && @parser.has_firstline?) ? flush_interval : nil
+        FileFeed.new(self, flush_interval: timer_flush_interval, flush_handler: @flush_handler)
+      end
+
+      # Parses and emits a batch for FileFeed and legacy watchers.
+      # The flush deadline is started by #parse_multilines for compatibility.
+      # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
+      def feed_lines(lines, tail_watcher)
+        emit(parse(lines, tail_watcher), tail_watcher)
+      end
+
+      def parse(lines, tail_watcher)
+        @parse_handler.call(lines, tail_watcher)
+      end
+
+      # Emits the events returned by #parse, with the tag of the file of the
+      # watcher which the lines were read from.
+      # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
+      def emit(es, tail_watcher)
+        return true if es.empty?
+
+        begin
+          @router_provider.call.emit_stream(tag_for(tail_watcher), es)
+        rescue Fluent::Plugin::Buffer::BufferOverflowError
+          return false
+        rescue
+          # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.
+          return true
+        end
+
+        return true
+      end
+
+      def flush_buffer(tw, buf)
+        buf.chomp!
+        @parser.parse(buf) { |time, record|
+          if time && record
+            record[@path_key] ||= tw.path unless @path_key.nil?
+            @router_provider.call.emit(tag_for(tw), time, record)
+          else
+            if @emit_unmatched_lines
+              record = { 'unmatched_line' => buf }
+              record[@path_key] ||= tw.path unless @path_key.nil?
+              @router_provider.call.emit(tag_for(tw), Fluent::EventTime.now, record)
+            end
+            @log.warn "got incomplete line at shutdown from #{tw.path}: #{buf.inspect}"
+          end
+        }
+      end
+
+      # Per-file state replacing TailWatcher::LineBufferTimerFlusher.
+      class FileFeed
+        attr_accessor :line_buffer
+
+        def initialize(line_feeder, flush_interval:, flush_handler:)
+          @line_feeder = line_feeder
+          @flush_interval = flush_interval
+          @flush_handler = flush_handler
+          @line_buffer = nil
+          @start = nil
+        end
+
+        # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
+        def feed_lines(lines, tail_watcher)
+          @line_feeder.feed_lines(lines, tail_watcher)
+        end
+
+        def on_notify(tail_watcher)
+          return unless @start
+
+          if Fluent::Clock.now - @start >= @flush_interval
+            flush_buffer(@line_buffer, tail_watcher) if @line_buffer
+            @line_buffer = nil
+            @start = nil
+          end
+        end
+
+        def close(tail_watcher)
+          return unless @line_buffer
+
+          flush_buffer(@line_buffer, tail_watcher)
+          @line_buffer = nil
+        end
+
+        def flush_buffer(buf, tail_watcher)
+          @flush_handler.call(tail_watcher, buf)
+        end
+
+        # Starts the flush deadline. Overrides that do not call super may call
+        # this method to preserve the old timer behavior.
+        def reset_timer
+          return unless @flush_interval
+
+          @start = Fluent::Clock.now
+        end
+      end
+
+      def convert_line_to_event(line, es, tail_watcher)
+        begin
+          line.chomp!  # remove \n
+          @parser.parse(line) { |time, record|
+            if time && record
+              record[@path_key] ||= tail_watcher.path unless @path_key.nil?
+              es.add(time, record)
+            else
+              if @emit_unmatched_lines
+                record = {'unmatched_line' => line}
+                record[@path_key] ||= tail_watcher.path unless @path_key.nil?
+                es.add(Fluent::EventTime.now, record)
+              end
+              @log.warn { "pattern not matched: #{line.inspect}" }
+            end
+          }
+        rescue => e
+          @log.warn 'invalid line found', file: tail_watcher.path, line: line, error: e.to_s
+          @log.debug_backtrace(e.backtrace)
+        end
+      end
+
+      def parse_singleline(lines, tail_watcher)
+        es = Fluent::MultiEventStream.new
+        lines.each { |line|
+          @convert_handler.call(line, es, tail_watcher)
+        }
+        es
+      end
+
+      # Parses multiline input and starts the flush deadline before parsing, as
+      # TailInput#parse_multilines did before LineFeeder was extracted.
+      def parse_multilines(lines, tail_watcher)
+        file_feed = tail_watcher.file_feed
+        lb = file_feed.line_buffer
+        es = Fluent::MultiEventStream.new
+        if @parser.has_firstline?
+          file_feed.reset_timer
+          lines.each { |line|
+            if @parser.firstline?(line)
+              if lb
+                @convert_handler.call(lb, es, tail_watcher)
+              end
+              lb = line
+            else
+              if lb.nil?
+                if @emit_unmatched_lines
+                  @convert_handler.call(line, es, tail_watcher)
+                end
+                @log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
+              else
+                lb << line
+              end
+            end
+          }
+        else
+          lb ||= ''
+          lines.each do |line|
+            lb << line
+            @parser.parse(lb) { |time, record|
+              if time && record
+                @convert_handler.call(lb, es, tail_watcher)
+                lb = ''
+              end
+            }
+          end
+        end
+        file_feed.line_buffer = lb
+        es
+      end
+
+      private
+
+      def tag_for(tw)
+        if @tag_prefix || @tag_suffix
+          @tag_prefix + tw.tag + @tag_suffix
+        else
+          @tag
+        end
+      end
+    end
+  end
+end