logstash-input-file 4.0.5 → 4.1.0

data/lib/filewatch/sincedb_value.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+
+module FileWatch
+  # Tracks the position and expiry of the offset of a file-of-interest
+  class SincedbValue
+    attr_reader :last_changed_at, :watched_file, :path_in_sincedb
+
+    def initialize(position, last_changed_at = nil, watched_file = nil)
+      @position = position # this is the value read from disk
+      @last_changed_at = last_changed_at
+      @watched_file = watched_file
+      touch if @last_changed_at.nil? || @last_changed_at.zero?
+    end
+
+    def add_path_in_sincedb(path)
+      @path_in_sincedb = path # can be nil
+      self
+    end
+
+    def last_changed_at_expires(duration)
+      @last_changed_at + duration
+    end
+
+    def position
+      # either the value from disk or the current wf position
+      @watched_file.nil? ? @position : @watched_file.bytes_read
+    end
+
+    def update_position(pos)
+      touch
+      if @watched_file.nil?
+        @position = pos
+      else
+        @watched_file.update_bytes_read(pos)
+      end
+    end
+
+    def increment_position(pos)
+      touch
+      if watched_file.nil?
+        @position += pos
+      else
+        @watched_file.increment_bytes_read(pos)
+      end
+    end
+
+    def set_watched_file(watched_file)
+      touch
+      @watched_file = watched_file
+    end
+
+    def touch
+      @last_changed_at = Time.now.to_f
+    end
+
+    def to_s
+      # consider serializing the watched_file state as well
+      "#{position} #{last_changed_at}".tap do |s|
+        if @watched_file.nil?
+          s.concat(" ").concat(@path_in_sincedb) unless @path_in_sincedb.nil?
+        else
+          s.concat(" ").concat(@watched_file.path)
+        end
+      end
+    end
+
+    def clear_watched_file
+      @watched_file = nil
+    end
+
+    def unset_watched_file
+      # cache the position
+      # we don't cache the path here because we know we are done with this file.
+      # either due via the `delete` handling
+      # or when read mode is done with a file.
+      # in the case of `delete` if the file was renamed then @watched_file is the
+      # watched_file of the previous path and the new path will be discovered and
+      # it should have the same inode as before.
+      # The key from the new watched_file should then locate this entry and we
+      # can resume from the cached position
+      return if @watched_file.nil?
+      wf = @watched_file
+      @watched_file = nil
+      @position = wf.bytes_read
+    end
+  end
+end

data/lib/filewatch/tail_mode/handlers/base.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+module FileWatch module TailMode module Handlers
+  class Base
+    include LogStash::Util::Loggable
+    attr_reader :sincedb_collection
+
+    def initialize(sincedb_collection, observer, settings)
+      @settings = settings
+      @sincedb_collection = sincedb_collection
+      @observer = observer
+    end
+
+    def handle(watched_file)
+      logger.debug("handling: #{watched_file.path}")
+      unless watched_file.has_listener?
+        watched_file.set_listener(@observer)
+      end
+      handle_specifically(watched_file)
+    end
+
+    def handle_specifically(watched_file)
+      # some handlers don't need to define this method
+    end
+
+    def update_existing_specifically(watched_file, sincedb_value)
+      # when a handler subclass does not implement this then do nothing
+    end
+
+    private
+
+    def read_to_eof(watched_file)
+      changed = false
+      # from a real config (has 102 file inputs)
+      # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously
+      # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF
+      # by allowing the user to specify a combo of `file_chunk_count` X `file_chunk_size`...
+      # we enable the pseudo parallel processing of each file.
+      # user also has the option to specify a low `stat_interval` and a very high `discover_interval`to respond
+      # quicker to changing files and not allowing too much content to build up before reading it.
+      @settings.file_chunk_count.times do
+        begin
+          data = watched_file.file_read(@settings.file_chunk_size)
+          lines = watched_file.buffer_extract(data)
+          logger.warn("read_to_eof: no delimiter found in current chunk") if lines.empty?
+          changed = true
+          lines.each do |line|
+            watched_file.listener.accept(line)
+            sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
+          end
+        rescue EOFError
+          # it only makes sense to signal EOF in "read" mode not "tail"
+          break
+        rescue Errno::EWOULDBLOCK, Errno::EINTR
+          watched_file.listener.error
+          break
+        rescue => e
+          logger.error("read_to_eof: general error reading #{watched_file.path}", "error" => e.inspect, "backtrace" => e.backtrace.take(4))
+          watched_file.listener.error
+          break
+        end
+      end
+      sincedb_collection.request_disk_flush if changed
+    end
+
+    def open_file(watched_file)
+      return true if watched_file.file_open?
+      logger.debug("opening #{watched_file.path}")
+      begin
+        watched_file.open
+      rescue
+        # don't emit this message too often. if a file that we can't
+        # read is changing a lot, we'll try to open it more often, and spam the logs.
+        now = Time.now.to_i
+        logger.warn("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
+        if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
+          logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
+          watched_file.last_open_warning_at = now
+        else
+          logger.debug("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
+        end
+        watched_file.watch # set it back to watch so we can try it again
+      end
+      if watched_file.file_open?
+        watched_file.listener.opened
+        true
+      else
+        false
+      end
+    end
+
+    def add_or_update_sincedb_collection(watched_file)
+      sincedb_value = @sincedb_collection.find(watched_file)
+      if sincedb_value.nil?
+        add_new_value_sincedb_collection(watched_file)
+      elsif sincedb_value.watched_file == watched_file
+        update_existing_sincedb_collection_value(watched_file, sincedb_value)
+      else
+        logger.warn? && logger.warn("mismatch on sincedb_value.watched_file, this should have been handled by Discoverer")
+      end
+      watched_file.initial_completed
+    end
+
+    def update_existing_sincedb_collection_value(watched_file, sincedb_value)
+      logger.debug("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
+      update_existing_specifically(watched_file, sincedb_value)
+    end
+
+    def add_new_value_sincedb_collection(watched_file)
+      sincedb_value = get_new_value_specifically(watched_file)
+      logger.debug("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position)
+      sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
+    end
+
+    def get_new_value_specifically(watched_file)
+      position = @settings.start_new_files_at == :beginning ? 0 : watched_file.last_stat_size
+      value = SincedbValue.new(position)
+      value.set_watched_file(watched_file)
+      watched_file.update_bytes_read(position)
+      value
+    end
+  end
+end end end

data/lib/filewatch/tail_mode/handlers/create.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+
+module FileWatch module TailMode module Handlers
+  class Create < Base
+    def handle_specifically(watched_file)
+      if open_file(watched_file)
+        add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
+      end
+    end
+
+    def update_existing_specifically(watched_file, sincedb_value)
+      # sincedb_value is the source of truth
+      position = sincedb_value.position
+      watched_file.update_bytes_read(position)
+    end
+  end
+end end end

data/lib/filewatch/tail_mode/handlers/create_initial.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+module FileWatch module TailMode module Handlers
+  class CreateInitial < Base
+    def handle_specifically(watched_file)
+      if open_file(watched_file)
+        add_or_update_sincedb_collection(watched_file)
+      end
+    end
+
+    def update_existing_specifically(watched_file, sincedb_value)
+      position = watched_file.last_stat_size
+      if @settings.start_new_files_at == :beginning
+        position = 0
+      end
+      logger.debug("update_existing_specifically - #{watched_file.path}: seeking to #{position}")
+      watched_file.update_bytes_read(position)
+      sincedb_value.update_position(position)
+    end
+  end
+end end end

data/lib/filewatch/tail_mode/handlers/delete.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+module FileWatch module TailMode module Handlers
+  class Delete < Base
+    def handle_specifically(watched_file)
+      watched_file.listener.deleted
+      sincedb_collection.unset_watched_file(watched_file)
+      watched_file.file_close
+    end
+  end
+end end end

data/lib/filewatch/tail_mode/handlers/grow.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+module FileWatch module TailMode module Handlers
+  class Grow < Base
+    def handle_specifically(watched_file)
+      watched_file.file_seek(watched_file.bytes_read)
+      logger.debug("reading to eof: #{watched_file.path}")
+      read_to_eof(watched_file)
+    end
+  end
+end end end

data/lib/filewatch/tail_mode/handlers/shrink.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+module FileWatch module TailMode module Handlers
+  class Shrink < Base
+    def handle_specifically(watched_file)
+      add_or_update_sincedb_collection(watched_file)
+      watched_file.file_seek(watched_file.bytes_read)
+      logger.debug("reading to eof: #{watched_file.path}")
+      read_to_eof(watched_file)
+    end
+
+    def update_existing_specifically(watched_file, sincedb_value)
+      # we have a match but size is smaller
+      # set all to zero
+      logger.debug("update_existing_specifically: #{watched_file.path}: was truncated seeking to beginning")
+      watched_file.update_bytes_read(0) if watched_file.bytes_read != 0
+      sincedb_value.update_position(0)
+    end
+  end
+end end end

data/lib/filewatch/tail_mode/handlers/timeout.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+module FileWatch module TailMode module Handlers
+  class Timeout < Base
+    def handle_specifically(watched_file)
+      watched_file.listener.timed_out
+      watched_file.file_close
+    end
+  end
+end end end

data/lib/filewatch/tail_mode/handlers/unignore.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+
+module FileWatch module TailMode module Handlers
+  class Unignore < Base
+    # a watched file can be put straight into the ignored state
+    # before any other handling has been done
+    # at a minimum we create or associate a sincedb value
+    def handle_specifically(watched_file)
+      add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
+    end
+
+    def get_new_value_specifically(watched_file)
+      # for file initially ignored their bytes_read was set to stat.size
+      # use this value not the `start_new_files_at` for the position
+      # logger.debug("get_new_value_specifically", "watched_file" => watched_file.inspect)
+      SincedbValue.new(watched_file.bytes_read).tap do |val|
+        val.set_watched_file(watched_file)
+      end
+    end
+
+    def update_existing_specifically(watched_file, sincedb_value)
+      # when this watched_file was ignored it had it bytes_read set to eof
+      # now the file has changed (watched_file.size_changed?)
+      # it has been put into the watched state so when it becomes active
+      # we will handle grow or shrink
+      # for now we seek to where we were before the file got ignored (grow)
+      # or to the start (shrink)
+      position = 0
+      if watched_file.shrunk?
+        watched_file.update_bytes_read(0)
+      else
+        position = watched_file.bytes_read
+      end
+      sincedb_value.update_position(position)
+    end
+  end
+end end end

data/lib/filewatch/tail_mode/processor.rb

@@ -0,0 +1,209 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require_relative "handlers/base"
+require_relative "handlers/create_initial"
+require_relative "handlers/create"
+require_relative "handlers/delete"
+require_relative "handlers/grow"
+require_relative "handlers/shrink"
+require_relative "handlers/timeout"
+require_relative "handlers/unignore"
+
+module FileWatch module TailMode
+  # Must handle
+  #   :create_initial - file is discovered and we have no record of it in the sincedb
+  #   :create - file is discovered and we have seen it before in the sincedb
+  #   :grow   - file has more content
+  #   :shrink - file has less content
+  #   :delete   - file can't be read
+  #   :timeout - file is closable
+  #   :unignore - file was ignored, but have now received new content
+  class Processor
+    include LogStash::Util::Loggable
+
+    attr_reader :watch, :deletable_filepaths
+
+    def initialize(settings)
+      @settings = settings
+      @deletable_filepaths = []
+    end
+
+    def add_watch(watch)
+      @watch = watch
+      self
+    end
+
+    def initialize_handlers(sincedb_collection, observer)
+      @create_initial = Handlers::CreateInitial.new(sincedb_collection, observer, @settings)
+      @create = Handlers::Create.new(sincedb_collection, observer, @settings)
+      @grow = Handlers::Grow.new(sincedb_collection, observer, @settings)
+      @shrink = Handlers::Shrink.new(sincedb_collection, observer, @settings)
+      @delete = Handlers::Delete.new(sincedb_collection, observer, @settings)
+      @timeout = Handlers::Timeout.new(sincedb_collection, observer, @settings)
+      @unignore = Handlers::Unignore.new(sincedb_collection, observer, @settings)
+    end
+
+    def create(watched_file)
+      @create.handle(watched_file)
+    end
+
+    def create_initial(watched_file)
+      @create_initial.handle(watched_file)
+    end
+
+    def grow(watched_file)
+      @grow.handle(watched_file)
+    end
+
+    def shrink(watched_file)
+      @shrink.handle(watched_file)
+    end
+
+    def delete(watched_file)
+      @delete.handle(watched_file)
+    end
+
+    def timeout(watched_file)
+      @timeout.handle(watched_file)
+    end
+
+    def unignore(watched_file)
+      @unignore.handle(watched_file)
+    end
+
+    def process_closed(watched_files)
+      logger.debug("Closed processing")
+      # Handles watched_files in the closed state.
+      # if its size changed it is put into the watched state
+      watched_files.select {|wf| wf.closed? }.each do |watched_file|
+        path = watched_file.path
+        begin
+          watched_file.restat
+          if watched_file.size_changed?
+            # if the closed file changed, move it to the watched state
+            # not to active state because we want to respect the active files window.
+            watched_file.watch
+          end
+        rescue Errno::ENOENT
+          # file has gone away or we can't read it anymore.
+          common_deleted_reaction(watched_file, "Closed")
+        rescue => e
+          common_error_reaction(path, e, "Closed")
+        end
+        break if watch.quit?
+      end
+    end
+
+    def process_ignored(watched_files)
+      logger.debug("Ignored processing")
+      # Handles watched_files in the ignored state.
+      # if its size changed:
+      #   put it in the watched state
+      #   invoke unignore
+      watched_files.select {|wf| wf.ignored? }.each do |watched_file|
+        path = watched_file.path
+        begin
+          watched_file.restat
+          if watched_file.size_changed?
+            watched_file.watch
+            unignore(watched_file)
+          end
+        rescue Errno::ENOENT
+          # file has gone away or we can't read it anymore.
+          common_deleted_reaction(watched_file, "Ignored")
+        rescue => e
+          common_error_reaction(path, e, "Ignored")
+        end
+        break if watch.quit?
+      end
+    end
+
+    def process_watched(watched_files)
+      logger.debug("Watched processing")
+      # Handles watched_files in the watched state.
+      # for a slice of them:
+      #   move to the active state
+      #   and we allow the block to open the file and create a sincedb collection record if needed
+      #   some have never been active and some have
+      #   those that were active before but are watched now were closed under constraint
+
+      # how much of the max active window is available
+      to_take = @settings.max_active - watched_files.count{|wf| wf.active?}
+      if to_take > 0
+        watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file|
+          path = watched_file.path
+          begin
+            watched_file.restat
+            watched_file.activate
+            if watched_file.initial?
+              create_initial(watched_file)
+            else
+              create(watched_file)
+            end
+          rescue Errno::ENOENT
+            # file has gone away or we can't read it anymore.
+            common_deleted_reaction(watched_file, "Watched")
+          rescue => e
+            common_error_reaction(path, e, "Watched")
+          end
+          break if watch.quit?
+        end
+      else
+        now = Time.now.to_i
+        if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL
+          waiting = watched_files.size - @settings.max_active
+          logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}")
+          watch.lastwarn_max_files = now
+        end
+      end
+    end
+
+    def process_active(watched_files)
+      logger.debug("Active processing")
+      # Handles watched_files in the active state.
+      # it has been read once - unless they were empty at the time
+      watched_files.select {|wf| wf.active? }.each do |watched_file|
+        path = watched_file.path
+        begin
+          watched_file.restat
+        rescue Errno::ENOENT
+          # file has gone away or we can't read it anymore.
+          common_deleted_reaction(watched_file, "Active")
+          next
+        rescue => e
+          common_error_reaction(path, e, "Active")
+          next
+        end
+        break if watch.quit?
+        if watched_file.grown?
+          logger.debug("Active - file grew: #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}")
+          grow(watched_file)
+        elsif watched_file.shrunk?
+          # we don't update the size here, its updated when we actually read
+          logger.debug("Active - file shrunk #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}")
+          shrink(watched_file)
+        else
+          # same size, do nothing
+        end
+        # can any active files be closed to make way for waiting files?
+        if watched_file.file_closable?
+          logger.debug("Watch each: active: file expired: #{path}")
+          timeout(watched_file)
+          watched_file.close
+        end
+      end
+    end
+
+    def common_deleted_reaction(watched_file, action)
+      # file has gone away or we can't read it anymore.
+      watched_file.unwatch
+      delete(watched_file)
+      deletable_filepaths << watched_file.path
+      logger.debug("#{action} - stat failed: #{watched_file.path}, removing from collection")
+    end
+
+    def common_error_reaction(path, error, action)
+      logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})")
+    end
+  end
+end end