logstash-input-file 4.0.5 → 4.1.0

data/lib/filewatch/read_mode/processor.rb

@@ -0,0 +1,117 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+require_relative "handlers/base"
+require_relative "handlers/read_file"
+require_relative "handlers/read_zip_file"
+
+module FileWatch module ReadMode
+  # Must handle
+  #   :read_file
+  #   :read_zip_file
+  class Processor
+    include LogStash::Util::Loggable
+
+    attr_reader :watch, :deletable_filepaths
+
+    def initialize(settings)
+      @settings = settings
+      @deletable_filepaths = []
+    end
+
+    def add_watch(watch)
+      @watch = watch
+      self
+    end
+
+    def initialize_handlers(sincedb_collection, observer)
+      @read_file = Handlers::ReadFile.new(sincedb_collection, observer, @settings)
+      @read_zip_file = Handlers::ReadZipFile.new(sincedb_collection, observer, @settings)
+    end
+
+    def read_file(watched_file)
+      @read_file.handle(watched_file)
+    end
+
+    def read_zip_file(watched_file)
+      @read_zip_file.handle(watched_file)
+    end
+
+    def process_closed(watched_files)
+      # do not process watched_files in the closed state.
+    end
+
+    def process_ignored(watched_files)
+      # do not process watched_files in the ignored state.
+    end
+
+    def process_watched(watched_files)
+      logger.debug("Watched processing")
+      # Handles watched_files in the watched state.
+      # for a slice of them:
+      #   move to the active state
+      #   should never have been active before
+      # how much of the max active window is available
+      to_take = @settings.max_active - watched_files.count{|wf| wf.active?}
+      if to_take > 0
+        watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file|
+          path = watched_file.path
+          begin
+            watched_file.restat
+            watched_file.activate
+          rescue Errno::ENOENT
+            common_deleted_reaction(watched_file, "Watched")
+            next
+          rescue => e
+            common_error_reaction(path, e, "Watched")
+            next
+          end
+          break if watch.quit?
+        end
+      else
+        now = Time.now.to_i
+        if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL
+          waiting = watched_files.size - @settings.max_active
+          logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}")
+          watch.lastwarn_max_files = now
+        end
+      end
+    end
+
+    def process_active(watched_files)
+      logger.debug("Active processing")
+      # Handles watched_files in the active state.
+      watched_files.select {|wf| wf.active? }.each do |watched_file|
+        path = watched_file.path
+        begin
+          watched_file.restat
+        rescue Errno::ENOENT
+          common_deleted_reaction(watched_file, "Active")
+          next
+        rescue => e
+          common_error_reaction(path, e, "Active")
+          next
+        end
+        break if watch.quit?
+
+        if watched_file.compressed?
+          read_zip_file(watched_file)
+        else
+          read_file(watched_file)
+        end
+        # handlers take care of closing and unwatching
+      end
+    end
+
+    def common_deleted_reaction(watched_file, action)
+      # file has gone away or we can't read it anymore.
+      watched_file.unwatch
+      deletable_filepaths << watched_file.path
+      logger.debug("#{action} - stat failed: #{watched_file.path}, removing from collection")
+    end
+
+    def common_error_reaction(path, error, action)
+      logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})")
+    end
+  end
+end end

data/lib/filewatch/settings.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+
+module FileWatch
+  class Settings
+    attr_reader :delimiter, :close_older, :ignore_older, :delimiter_byte_size
+    attr_reader :max_active, :max_warn_msg, :lastwarn_max_files
+    attr_reader :sincedb_write_interval, :stat_interval, :discover_interval
+    attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size
+    attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration
+    attr_reader :file_sort_by, :file_sort_direction
+
+    def self.from_options(opts)
+      new.add_options(opts)
+    end
+
+    def self.days_to_seconds(days)
+      (24 * 3600) * days.to_f
+    end
+
+    def initialize
+      defaults = {
+        :delimiter => "\n",
+        :file_chunk_size => FILE_READ_SIZE,
+        :max_active => 4095,
+        :file_chunk_count => FIXNUM_MAX,
+        :sincedb_clean_after => 14,
+        :exclude => [],
+        :stat_interval => 1,
+        :discover_interval => 5,
+        :file_sort_by => "last_modified",
+        :file_sort_direction => "asc",
+      }
+      @opts = {}
+      @lastwarn_max_files = 0
+      add_options(defaults)
+    end
+
+    def add_options(opts)
+      @opts.update(opts)
+      self.max_open_files = @opts[:max_active]
+      @delimiter = @opts[:delimiter]
+      @delimiter_byte_size = @delimiter.bytesize
+      @file_chunk_size = @opts[:file_chunk_size]
+      @close_older = @opts[:close_older]
+      @ignore_older = @opts[:ignore_older]
+      @sincedb_write_interval = @opts[:sincedb_write_interval]
+      @stat_interval = @opts[:stat_interval]
+      @discover_interval = @opts[:discover_interval]
+      @exclude = Array(@opts[:exclude])
+      @start_new_files_at = @opts[:start_new_files_at]
+      @file_chunk_count = @opts[:file_chunk_count]
+      @sincedb_path = @opts[:sincedb_path]
+      @sincedb_write_interval = @opts[:sincedb_write_interval]
+      @sincedb_expiry_duration =  self.class.days_to_seconds(@opts.fetch(:sincedb_clean_after, 14))
+      @file_sort_by = @opts[:file_sort_by]
+      @file_sort_direction = @opts[:file_sort_direction]
+      self
+    end
+
+    def max_open_files=(value)
+      val = value.to_i
+      val = 4095 if value.nil? || val <= 0
+      @max_warn_msg = "Reached open files limit: #{val}, set by the 'max_open_files' option or default"
+      @max_active = val
+    end
+  end
+end

data/lib/filewatch/sincedb_collection.rb

@@ -0,0 +1,215 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+module FileWatch
+  # this KV collection has a watched_file storage_key (an InodeStruct) as the key
+  # and a SincedbValue as the value.
+  # the SincedbValues are built by reading the sincedb file.
+  class SincedbCollection
+    include LogStash::Util::Loggable
+
+    attr_reader :path
+    attr_writer :serializer
+
+    def initialize(settings)
+      @settings = settings
+      @sincedb_last_write = 0
+      @sincedb = {}
+      @serializer = SincedbRecordSerializer.new(@settings.sincedb_expiry_duration)
+      @path = Pathname.new(@settings.sincedb_path)
+      @write_method = LogStash::Environment.windows? || @path.chardev? || @path.blockdev? ? method(:non_atomic_write) : method(:atomic_write)
+      @full_path = @path.to_path
+      FileUtils.touch(@full_path)
+    end
+
+    def request_disk_flush
+      now = Time.now.to_i
+      delta = now - @sincedb_last_write
+      if delta >= @settings.sincedb_write_interval
+        logger.debug("writing sincedb (delta since last write = #{delta})")
+        sincedb_write(now)
+      end
+    end
+
+    def write(reason=nil)
+      logger.debug("caller requested sincedb write (#{reason})")
+      sincedb_write
+    end
+
+    def open
+      @time_sdb_opened = Time.now.to_f
+      begin
+        path.open do |file|
+          logger.debug("open: reading from #{path}")
+          @serializer.deserialize(file) do |key, value|
+            logger.debug("open: importing ... '#{key}' => '#{value}'")
+            set_key_value(key, value)
+          end
+        end
+        logger.debug("open: count of keys read: #{@sincedb.keys.size}")
+      rescue => e
+        #No existing sincedb to load
+        logger.debug("open: error: #{path}: #{e.inspect}")
+      end
+
+    end
+
+    def associate(watched_file)
+      logger.debug("associate: finding: #{watched_file.path}")
+      sincedb_value = find(watched_file)
+      if sincedb_value.nil?
+        # sincedb has no record of this inode
+        # and due to the window handling of many files
+        # this file may not be opened in this session.
+        # a new value will be added when the file is opened
+        return
+      end
+      if sincedb_value.watched_file.nil?
+        # not associated
+        if sincedb_value.path_in_sincedb.nil?
+          # old v1 record, assume its the same file
+          handle_association(sincedb_value, watched_file)
+          return
+        end
+        if sincedb_value.path_in_sincedb == watched_file.path
+          # the path on disk is the same as discovered path
+          # and the inode is the same.
+          handle_association(sincedb_value, watched_file)
+          return
+        end
+        # the path on disk is different from discovered unassociated path
+        # but they have the same key (inode)
+        # treat as a new file, a new value will be added when the file is opened
+        logger.debug("associate: matched but allocated to another - #{sincedb_value}")
+        sincedb_value.clear_watched_file
+        delete(watched_file.sincedb_key)
+        return
+      end
+      if sincedb_value.watched_file.equal?(watched_file) # pointer equals
+        logger.debug("associate: already associated - #{sincedb_value}, for path: #{watched_file.path}")
+        return
+      end
+      # sincedb_value.watched_file is not the discovered watched_file but they have the same key (inode)
+      # this means that the filename was changed during this session.
+      # logout the history of the old sincedb_value and remove it
+      # a new value will be added when the file is opened
+      # TODO notify about done-ness of old sincedb_value and watched_file
+      old_watched_file = sincedb_value.watched_file
+      sincedb_value.clear_watched_file
+      if logger.debug?
+        logger.debug("associate: matched but allocated to another - #{sincedb_value}")
+        logger.debug("associate: matched but allocated to another - old watched_file history - #{old_watched_file.recent_state_history.join(', ')}")
+        logger.debug("associate: matched but allocated to another - DELETING value at key `#{old_watched_file.sincedb_key}`")
+      end
+      delete(old_watched_file.sincedb_key)
+    end
+
+    def find(watched_file)
+      get(watched_file.sincedb_key).tap do |obj|
+        logger.debug("find for path: #{watched_file.path}, found: '#{!obj.nil?}'")
+      end
+    end
+
+    def member?(key)
+      @sincedb.member?(key)
+    end
+
+    def get(key)
+      @sincedb[key]
+    end
+
+    def delete(key)
+      @sincedb.delete(key)
+    end
+
+    def last_read(key)
+      @sincedb[key].position
+    end
+
+    def rewind(key)
+      @sincedb[key].update_position(0)
+    end
+
+    def store_last_read(key, last_read)
+      @sincedb[key].update_position(last_read)
+    end
+
+    def increment(key, amount)
+      @sincedb[key].increment_position(amount)
+    end
+
+    def set_watched_file(key, watched_file)
+      @sincedb[key].set_watched_file(watched_file)
+    end
+
+    def unset_watched_file(watched_file)
+      return unless member?(watched_file.sincedb_key)
+      get(watched_file.sincedb_key).unset_watched_file
+    end
+
+    def clear
+      @sincedb.clear
+    end
+
+    def keys
+      @sincedb.keys
+    end
+
+    def set(key, value)
+      @sincedb[key] = value
+      value
+    end
+
+    def watched_file_unset?(key)
+      return false unless member?(key)
+      get(key).watched_file.nil?
+    end
+
+    private
+
+    def handle_association(sincedb_value, watched_file)
+      watched_file.update_bytes_read(sincedb_value.position)
+      sincedb_value.set_watched_file(watched_file)
+      watched_file.initial_completed
+      watched_file.ignore if watched_file.all_read?
+    end
+
+    def set_key_value(key, value)
+      if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration)
+        logger.debug("open: setting #{key.inspect} to #{value.inspect}")
+        set(key, value)
+      else
+        logger.debug("open: record has expired, skipping: #{key.inspect} #{value.inspect}")
+      end
+    end
+
+    def sincedb_write(time = Time.now.to_i)
+      logger.debug("sincedb_write: to: #{path}")
+      begin
+        @write_method.call
+        @serializer.expired_keys.each do |key|
+          @sincedb[key].unset_watched_file
+          delete(key)
+          logger.debug("sincedb_write: cleaned", "key" => "'#{key}'")
+        end
+        @sincedb_last_write = time
+      rescue Errno::EACCES
+        # no file handles free perhaps
+        # maybe it will work next time
+        logger.debug("sincedb_write: error: #{path}: #{$!}")
+      end
+    end
+
+    def atomic_write
+      FileHelper.write_atomically(@full_path) do |io|
+        @serializer.serialize(@sincedb, io)
+      end
+    end
+
+    def non_atomic_write
+      IO.open(@full_path, 0) do |io|
+        @serializer.serialize(@sincedb, io)
+      end
+    end
+  end
+end

data/lib/filewatch/sincedb_record_serializer.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+
+module FileWatch
+  class SincedbRecordSerializer
+
+    attr_reader :expired_keys
+
+    def initialize(sincedb_value_expiry)
+      @sincedb_value_expiry = sincedb_value_expiry
+      @expired_keys = []
+    end
+
+    def update_sincedb_value_expiry_from_days(days)
+      @sincedb_value_expiry = Settings.days_to_seconds(days)
+    end
+
+    def serialize(db, io, as_of = Time.now.to_f)
+      @expired_keys.clear
+      db.each do |key, value|
+        if as_of > value.last_changed_at_expires(@sincedb_value_expiry)
+          @expired_keys << key
+          next
+        end
+        io.write(serialize_record(key, value))
+      end
+    end
+
+    def deserialize(io)
+      io.each do |record|
+        yield deserialize_record(record) #.tap{|val| STDERR.puts val}
+      end
+    end
+
+    def serialize_record(k, v)
+      # effectively InodeStruct#to_s SincedbValue#to_s
+      "#{k} #{v}\n"
+    end
+
+    def deserialize_record(record)
+      return [] if record.nil? || record.empty?
+      parts = record.split(" ")
+      parse_line_v2(parts) || parse_line_v1(parts)
+    end
+
+    private
+
+    def parse_line_v2(parts)
+      # new format e.g. 2977152 1 4 94 1519319662.852678 'path/to/file'
+      # do we want to store the last known state of the watched file too?
+      return false if parts.size < 5
+      inode_struct = prepare_inode_struct(parts)
+      pos = parts.shift.to_i
+      expires_at = Float(parts.shift) # this is like Time.now.to_f
+      path_in_sincedb = parts.shift
+      value = SincedbValue.new(pos, expires_at).add_path_in_sincedb(path_in_sincedb)
+      [inode_struct, value]
+    end
+
+    def parse_line_v1(parts)
+      # old inode based e.g. 2977152 1 4 94
+      inode_struct = prepare_inode_struct(parts)
+      pos = parts.shift.to_i
+      [inode_struct, SincedbValue.new(pos)]
+    end
+
+    def prepare_inode_struct(parts)
+      InodeStruct.new(parts.shift, *parts.shift(2).map(&:to_i))
+    end
+  end
+end