fluentd 1.20.0-aarch64-mingw-ucrt

data/lib/fluent/plugin/in_tail/line_feeder.rb

@@ -0,0 +1,230 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'fluent/event'
+require 'fluent/plugin/buffer'
+require 'fluent/clock'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    # LineFeeder provides the built-in line parsing and emitting for TailInput.
+    # To preserve compatibility with TailInput subclasses, TailInput passes its
+    # line-processing methods as handlers. This keeps subclass overrides in the
+    # call path, and super reaches this built-in implementation.
+    class LineFeeder
+      def initialize(parser:, router_provider:, log:, tag:, tag_prefix:, tag_suffix:, path_key:, emit_unmatched_lines:, multiline_mode:, parse_handler: nil, convert_handler: nil, flush_handler: nil)
+        @parser = parser
+        @router_provider = router_provider
+        @log = log
+        @tag = tag
+        @tag_prefix = tag_prefix
+        @tag_suffix = tag_suffix
+        @path_key = path_key
+        @emit_unmatched_lines = emit_unmatched_lines
+        @multiline_mode = multiline_mode
+        @parse_handler = parse_handler || if multiline_mode
+                                           method(:parse_multilines)
+                                         else
+                                           method(:parse_singleline)
+                                         end
+        @convert_handler = convert_handler || method(:convert_line_to_event)
+        @flush_handler = flush_handler || method(:flush_buffer)
+      end
+
+      # Builds a per-file feed. The flush interval applies only to multiline
+      # parsers with a firstline.
+      def new_file_feed(flush_interval: nil)
+        timer_flush_interval = (@multiline_mode && @parser.has_firstline?) ? flush_interval : nil
+        FileFeed.new(self, flush_interval: timer_flush_interval, flush_handler: @flush_handler)
+      end
+
+      # Parses and emits a batch for FileFeed and legacy watchers.
+      # The flush deadline is started by #parse_multilines for compatibility.
+      # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
+      def feed_lines(lines, tail_watcher)
+        emit(parse(lines, tail_watcher), tail_watcher)
+      end
+
+      def parse(lines, tail_watcher)
+        @parse_handler.call(lines, tail_watcher)
+      end
+
+      # Emits the events returned by #parse, with the tag of the file of the
+      # watcher which the lines were read from.
+      # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
+      def emit(es, tail_watcher)
+        return true if es.empty?
+
+        begin
+          @router_provider.call.emit_stream(tag_for(tail_watcher), es)
+        rescue Fluent::Plugin::Buffer::BufferOverflowError
+          return false
+        rescue
+          # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.
+          return true
+        end
+
+        return true
+      end
+
+      def flush_buffer(tw, buf)
+        buf.chomp!
+        @parser.parse(buf) { |time, record|
+          if time && record
+            record[@path_key] ||= tw.path unless @path_key.nil?
+            @router_provider.call.emit(tag_for(tw), time, record)
+          else
+            if @emit_unmatched_lines
+              record = { 'unmatched_line' => buf }
+              record[@path_key] ||= tw.path unless @path_key.nil?
+              @router_provider.call.emit(tag_for(tw), Fluent::EventTime.now, record)
+            end
+            @log.warn "got incomplete line at shutdown from #{tw.path}: #{buf.inspect}"
+          end
+        }
+      end
+
+      # Per-file state replacing TailWatcher::LineBufferTimerFlusher.
+      class FileFeed
+        attr_accessor :line_buffer
+
+        def initialize(line_feeder, flush_interval:, flush_handler:)
+          @line_feeder = line_feeder
+          @flush_interval = flush_interval
+          @flush_handler = flush_handler
+          @line_buffer = nil
+          @start = nil
+        end
+
+        # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
+        def feed_lines(lines, tail_watcher)
+          @line_feeder.feed_lines(lines, tail_watcher)
+        end
+
+        def on_notify(tail_watcher)
+          return unless @start
+
+          if Fluent::Clock.now - @start >= @flush_interval
+            flush_buffer(@line_buffer, tail_watcher) if @line_buffer
+            @line_buffer = nil
+            @start = nil
+          end
+        end
+
+        def close(tail_watcher)
+          return unless @line_buffer
+
+          flush_buffer(@line_buffer, tail_watcher)
+          @line_buffer = nil
+        end
+
+        def flush_buffer(buf, tail_watcher)
+          @flush_handler.call(tail_watcher, buf)
+        end
+
+        # Starts the flush deadline. Overrides that do not call super may call
+        # this method to preserve the old timer behavior.
+        def reset_timer
+          return unless @flush_interval
+
+          @start = Fluent::Clock.now
+        end
+      end
+
+      def convert_line_to_event(line, es, tail_watcher)
+        begin
+          line.chomp!  # remove \n
+          @parser.parse(line) { |time, record|
+            if time && record
+              record[@path_key] ||= tail_watcher.path unless @path_key.nil?
+              es.add(time, record)
+            else
+              if @emit_unmatched_lines
+                record = {'unmatched_line' => line}
+                record[@path_key] ||= tail_watcher.path unless @path_key.nil?
+                es.add(Fluent::EventTime.now, record)
+              end
+              @log.warn { "pattern not matched: #{line.inspect}" }
+            end
+          }
+        rescue => e
+          @log.warn 'invalid line found', file: tail_watcher.path, line: line, error: e.to_s
+          @log.debug_backtrace(e.backtrace)
+        end
+      end
+
+      def parse_singleline(lines, tail_watcher)
+        es = Fluent::MultiEventStream.new
+        lines.each { |line|
+          @convert_handler.call(line, es, tail_watcher)
+        }
+        es
+      end
+
+      # Parses multiline input and starts the flush deadline before parsing, as
+      # TailInput#parse_multilines did before LineFeeder was extracted.
+      def parse_multilines(lines, tail_watcher)
+        file_feed = tail_watcher.file_feed
+        lb = file_feed.line_buffer
+        es = Fluent::MultiEventStream.new
+        if @parser.has_firstline?
+          file_feed.reset_timer
+          lines.each { |line|
+            if @parser.firstline?(line)
+              if lb
+                @convert_handler.call(lb, es, tail_watcher)
+              end
+              lb = line
+            else
+              if lb.nil?
+                if @emit_unmatched_lines
+                  @convert_handler.call(line, es, tail_watcher)
+                end
+                @log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
+              else
+                lb << line
+              end
+            end
+          }
+        else
+          lb ||= ''
+          lines.each do |line|
+            lb << line
+            @parser.parse(lb) { |time, record|
+              if time && record
+                @convert_handler.call(lb, es, tail_watcher)
+                lb = ''
+              end
+            }
+          end
+        end
+        file_feed.line_buffer = lb
+        es
+      end
+
+      private
+
+      def tag_for(tw)
+        if @tag_prefix || @tag_suffix
+          @tag_prefix + tw.tag + @tag_suffix
+        else
+          @tag
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -0,0 +1,269 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    class PositionFile
+      UNWATCHED_POSITION = 0xffffffffffffffff
+      POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
+
+      def self.load(file, follow_inodes, existing_targets, logger:)
+        pf = new(file, follow_inodes, logger: logger)
+        pf.load(existing_targets)
+        pf
+      end
+
+      def initialize(file, follow_inodes, logger: nil)
+        @file = file
+        @logger = logger
+        @file_mutex = Mutex.new
+        @map = {}
+        @follow_inodes = follow_inodes
+      end
+
+      def [](target_info)
+        if m = @map[@follow_inodes ? target_info.ino : target_info.path]
+          return m
+        end
+
+        @file_mutex.synchronize {
+          @file.seek(0, IO::SEEK_END)
+          seek = @file.pos + target_info.path.bytesize + 1
+          @file.write "#{target_info.path}\t0000000000000000\t0000000000000000\n"
+          if @follow_inodes
+            @map[target_info.ino] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+          else
+            @map[target_info.path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+          end
+        }
+      end
+
+      def unwatch_removed_targets(existing_targets)
+        @map.reject { |key, entry|
+          existing_targets.key?(key)
+        }.each_key { |key|
+          unwatch_key(key)
+        }
+      end
+
+      def unwatch(target_info)
+        unwatch_key(@follow_inodes ? target_info.ino : target_info.path)
+      end
+
+      def load(existing_targets = nil)
+        compact(existing_targets)
+
+        map = {}
+        @file_mutex.synchronize do
+          @file.pos = 0
+
+          @file.each_line do |line|
+            m = POSITION_FILE_ENTRY_REGEX.match(line)
+            next if m.nil?
+
+            path = m[1]
+            pos = m[2].to_i(16)
+            ino = m[3].to_i(16)
+            seek = @file.pos - line.bytesize + path.bytesize + 1
+            if @follow_inodes
+              map[ino] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
+            else
+              map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
+            end
+          end
+        end
+
+        @map = map
+      end
+
+      # This method is similar to #compact but it tries to get less lock to avoid a lock contention
+      def try_compact
+        last_modified = nil
+        size = nil
+
+        @file_mutex.synchronize do
+          size = @file.size
+          last_modified = @file.mtime
+        end
+
+        entries = fetch_compacted_entries
+        @logger&.debug "Compacted entries: ", entries.keys
+
+        @file_mutex.synchronize do
+          if last_modified == @file.mtime && size == @file.size
+            @file.pos = 0
+            @file.truncate(0)
+            @file.write(entries.values.map(&:to_entry_fmt).join)
+
+            # entry contains path/ino key and value.
+            entries.each do |key, val|
+              if (m = @map[key])
+                m.seek = val.seek
+              end
+            end
+          else
+            # skip
+          end
+        end
+      end
+
+      private
+
+      def unwatch_key(key)
+        if (entry = @map.delete(key))
+          entry.update_pos(UNWATCHED_POSITION)
+        end
+      end
+
+      def compact(existing_targets = nil)
+        @file_mutex.synchronize do
+          entries = fetch_compacted_entries
+          @logger&.debug "Compacted entries: ", entries.keys
+
+          if existing_targets
+            entries = remove_deleted_files_entries(entries, existing_targets)
+            @logger&.debug "Remove missing entries.",
+                           existing_targets: existing_targets.keys,
+                           entries_after_removing: entries.keys
+          end
+
+          @file.pos = 0
+          @file.truncate(0)
+          @file.write(entries.values.map(&:to_entry_fmt).join)
+        end
+      end
+
+      def fetch_compacted_entries
+        entries = {}
+
+        @file.pos = 0
+        file_pos = 0
+        @file.each_line do |line|
+          m = POSITION_FILE_ENTRY_REGEX.match(line)
+          if m.nil?
+            @logger.warn "Unparsable line in pos_file: #{line}" if @logger
+            next
+          end
+
+          path = m[1]
+          pos = m[2].to_i(16)
+          ino = m[3].to_i(16)
+          if pos == UNWATCHED_POSITION
+            @logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
+          else
+            if @follow_inodes
+              @logger&.warn("#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}") if entries.include?(ino)
+              entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
+            else
+              @logger&.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if entries.include?(path)
+              entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
+            end
+            file_pos += line.size
+          end
+        end
+
+        entries
+      end
+
+      def remove_deleted_files_entries(existent_entries, existing_targets)
+        existent_entries.select { |path_or_ino|
+          existing_targets.key?(path_or_ino)
+        }
+      end
+    end
+
+    Entry = Struct.new(:path, :pos, :ino, :seek) do
+      POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
+
+      def to_entry_fmt
+        POSITION_FILE_ENTRY_FORMAT % [path, pos, ino]
+      end
+    end
+
+    # pos               inode
+    # ffffffffffffffff\tffffffffffffffff\n
+    class FilePositionEntry
+      POS_SIZE = 16
+      INO_OFFSET = 17
+      INO_SIZE = 16
+      LN_OFFSET = 33
+      SIZE = 34
+
+      def initialize(file, file_mutex, seek, pos, inode)
+        @file = file
+        @file_mutex = file_mutex
+        @seek = seek
+        @pos = pos
+        @inode = inode
+      end
+
+      attr_writer :seek
+
+      def update(ino, pos)
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x\t%016x" % [pos, ino]
+        }
+        @pos = pos
+        @inode = ino
+      end
+
+      def update_pos(pos)
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x" % pos
+        }
+        @pos = pos
+      end
+
+      def read_inode
+        @inode
+      end
+
+      def read_pos
+        @pos
+      end
+    end
+
+    class MemoryPositionEntry
+      def initialize
+        @pos = 0
+        @inode = 0
+      end
+
+      def update(ino, pos)
+        @inode = ino
+        @pos = pos
+      end
+
+      def update_pos(pos)
+        @pos = pos
+      end
+
+      def read_pos
+        @pos
+      end
+
+      def read_inode
+        @inode
+      end
+    end
+
+    TargetInfo = Struct.new(:path, :ino)
+  end
+end

data/lib/fluent/plugin/in_tail/stat_watcher.rb

@@ -0,0 +1,52 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'cool.io'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    class StatWatcher < Coolio::StatWatcher
+      def initialize(path, log, &callback)
+        @callback = callback
+        @log = log
+        super(path)
+      end
+
+      def on_change(prev, cur)
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+
+    class TimerTrigger < Coolio::TimerWatcher
+      def initialize(interval, log, repeat: true, &callback)
+        @log = log
+        @callback = callback
+        super(interval, repeat)
+      end
+
+      def on_timer
+        @callback.call
+      rescue => e
+        @log.error e.to_s
+        @log.error_backtrace
+      end
+    end
+  end
+end