logstash-input-file 4.1.3 → 4.1.4

data/lib/filewatch/read_mode/processor.rb

@@ -39,16 +39,16 @@ module FileWatch module ReadMode
       @read_zip_file.handle(watched_file)
     end
 
-    def process_closed(watched_files)
-      # do not process watched_files in the closed state.
+    def process_all_states(watched_files)
+      process_watched(watched_files)
+      return if watch.quit?
+      process_active(watched_files)
     end
 
-    def process_ignored(watched_files)
-      # do not process watched_files in the ignored state.
-    end
+    private
 
     def process_watched(watched_files)
-      logger.debug("Watched processing")
+      logger.trace("Watched processing")
       # Handles watched_files in the watched state.
       # for a slice of them:
       #   move to the active state
@@ -81,7 +81,7 @@ module FileWatch module ReadMode
     end
 
     def process_active(watched_files)
-      logger.debug("Active processing")
+      logger.trace("Active processing")
       # Handles watched_files in the active state.
       watched_files.select {|wf| wf.active? }.each do |watched_file|
         path = watched_file.path
@@ -109,7 +109,7 @@ module FileWatch module ReadMode
       # file has gone away or we can't read it anymore.
       watched_file.unwatch
       deletable_filepaths << watched_file.path
-      logger.debug("#{action} - stat failed: #{watched_file.path}, removing from collection")
+      logger.trace("#{action} - stat failed: #{watched_file.path}, removing from collection")
     end
 
     def common_error_reaction(path, error, action)

data/lib/filewatch/settings.rb

@@ -21,8 +21,8 @@ module FileWatch
       defaults = {
         :delimiter => "\n",
         :file_chunk_size => FILE_READ_SIZE,
-        :max_active => 4095,
-        :file_chunk_count => FIXNUM_MAX,
+        :max_open_files => 4095,
+        :file_chunk_count => MAX_ITERATIONS,
         :sincedb_clean_after => 14,
         :exclude => [],
         :stat_interval => 1,
@@ -37,7 +37,7 @@ module FileWatch
 
     def add_options(opts)
       @opts.update(opts)
-      self.max_open_files = @opts[:max_active]
+      self.max_open_files = @opts[:max_open_files]
       @delimiter = @opts[:delimiter]
       @delimiter_byte_size = @delimiter.bytesize
       @file_chunk_size = @opts[:file_chunk_size]

data/lib/filewatch/sincedb_collection.rb

@@ -39,7 +39,7 @@ module FileWatch
     end
 
     def write(reason=nil)
-      logger.debug("caller requested sincedb write (#{reason})")
+      logger.trace("caller requested sincedb write (#{reason})")
       sincedb_write
     end
 
@@ -47,73 +47,72 @@ module FileWatch
       @time_sdb_opened = Time.now.to_f
       begin
         path.open do |file|
-          logger.debug("open: reading from #{path}")
+          logger.trace("open: reading from #{path}")
           @serializer.deserialize(file) do |key, value|
-            logger.debug("open: importing ... '#{key}' => '#{value}'")
+            logger.trace("open: importing ... '#{key}' => '#{value}'")
             set_key_value(key, value)
           end
         end
-        logger.debug("open: count of keys read: #{@sincedb.keys.size}")
+        logger.trace("open: count of keys read: #{@sincedb.keys.size}")
       rescue => e
         #No existing sincedb to load
-        logger.debug("open: error: #{path}: #{e.inspect}")
+        logger.trace("open: error: #{path}: #{e.inspect}")
       end
     end
 
     def associate(watched_file)
-      logger.debug("associate: finding: #{watched_file.path}")
+      logger.trace("associate: finding", "inode" => watched_file.sincedb_key.inode, "path" => watched_file.path)
       sincedb_value = find(watched_file)
       if sincedb_value.nil?
         # sincedb has no record of this inode
         # and due to the window handling of many files
         # this file may not be opened in this session.
         # a new value will be added when the file is opened
-        return
+        logger.trace("associate: unmatched")
+        return true
       end
+      logger.trace("associate: found sincedb record", "filename" => watched_file.filename, "sincedb key" => watched_file.sincedb_key,"sincedb_value" => sincedb_value)
       if sincedb_value.watched_file.nil?
         # not associated
         if sincedb_value.path_in_sincedb.nil?
-          # old v1 record, assume its the same file
           handle_association(sincedb_value, watched_file)
-          return
+          logger.trace("associate: inode matched but no path in sincedb")
+          return true
         end
         if sincedb_value.path_in_sincedb == watched_file.path
           # the path on disk is the same as discovered path
           # and the inode is the same.
           handle_association(sincedb_value, watched_file)
-          return
+          logger.trace("associate: inode and path matched")
+          return true
         end
         # the path on disk is different from discovered unassociated path
         # but they have the same key (inode)
         # treat as a new file, a new value will be added when the file is opened
-        logger.debug("associate: matched but allocated to another - #{sincedb_value}")
         sincedb_value.clear_watched_file
         delete(watched_file.sincedb_key)
-        return
+        logger.trace("associate: matched but allocated to another")
+        return true
       end
       if sincedb_value.watched_file.equal?(watched_file) # pointer equals
-        logger.debug("associate: already associated - #{sincedb_value}, for path: #{watched_file.path}")
-        return
+        logger.trace("associate: already associated")
+        return true
       end
-      # sincedb_value.watched_file is not the discovered watched_file but they have the same key (inode)
-      # this means that the filename was changed during this session.
-      # logout the history of the old sincedb_value and remove it
-      # a new value will be added when the file is opened
-      # TODO notify about done-ness of old sincedb_value and watched_file
-      old_watched_file = sincedb_value.watched_file
-      sincedb_value.clear_watched_file
-      if logger.debug?
-        logger.debug("associate: matched but allocated to another - #{sincedb_value}")
-        logger.debug("associate: matched but allocated to another - old watched_file history - #{old_watched_file.recent_state_history.join(', ')}")
-        logger.debug("associate: matched but allocated to another - DELETING value at key `#{old_watched_file.sincedb_key}`")
-      end
-      delete(old_watched_file.sincedb_key)
+      # sincedb_value.watched_file is not this discovered watched_file but they have the same key (inode)
+      # this means that the filename path was changed during this session.
+      # renamed file can be discovered...
+      #   before the original is detected as deleted: state is `active`
+      #   after the original is detected as deleted but before it is actually deleted: state is `delayed_delete`
+      #   after the original is deleted
+      # are not yet in the delete phase, let this play out
+      existing_watched_file = sincedb_value.watched_file
+      logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename", "this watched_file details" => watched_file.details, "other watched_file details" => existing_watched_file.details)
+      watched_file.rotation_in_progress
+      true
     end
 
     def find(watched_file)
-      get(watched_file.sincedb_key).tap do |obj|
-        logger.debug("find for path: #{watched_file.path}, found: '#{!obj.nil?}'")
-      end
+      get(watched_file.sincedb_key)
     end
 
     def member?(key)
@@ -124,6 +123,11 @@ module FileWatch
       @sincedb[key]
     end
 
+    def set(key, value)
+      @sincedb[key] = value
+      value
+    end
+
     def delete(key)
       @sincedb.delete(key)
     end
@@ -144,11 +148,23 @@ module FileWatch
       @sincedb[key].set_watched_file(watched_file)
     end
 
-    def unset_watched_file(watched_file)
+    def watched_file_deleted(watched_file)
       return unless member?(watched_file.sincedb_key)
       get(watched_file.sincedb_key).unset_watched_file
     end
 
+    def store_last_read(key, pos)
+      @sincedb[key].update_position(pos)
+    end
+
+    def clear_watched_file(key)
+      @sincedb[key].clear_watched_file
+    end
+
+    def reading_completed(key)
+      @sincedb[key].reading_completed
+    end
+
     def clear
       @sincedb.clear
     end
@@ -157,11 +173,6 @@ module FileWatch
       @sincedb.keys
     end
 
-    def set(key, value)
-      @sincedb[key] = value
-      value
-    end
-
     def watched_file_unset?(key)
       return false unless member?(key)
       get(key).watched_file.nil?
@@ -182,33 +193,36 @@ module FileWatch
       watched_file.update_bytes_read(sincedb_value.position)
       sincedb_value.set_watched_file(watched_file)
       watched_file.initial_completed
-      watched_file.ignore if watched_file.all_read?
+      if watched_file.all_read?
+        watched_file.ignore
+        logger.trace("handle_association fully read, ignoring.....", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
+      end
     end
 
     def set_key_value(key, value)
       if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration)
-        logger.debug("open: setting #{key.inspect} to #{value.inspect}")
+        logger.trace("open: setting #{key.inspect} to #{value.inspect}")
         set(key, value)
       else
-        logger.debug("open: record has expired, skipping: #{key.inspect} #{value.inspect}")
+        logger.trace("open: record has expired, skipping: #{key.inspect} #{value.inspect}")
       end
     end
 
     def sincedb_write(time = Time.now.to_i)
-      logger.debug("sincedb_write: to: #{path}")
+      logger.trace("sincedb_write: to: #{path}")
       begin
         @write_method.call
         @serializer.expired_keys.each do |key|
           @sincedb[key].unset_watched_file
           delete(key)
-          logger.debug("sincedb_write: cleaned", "key" => "'#{key}'")
+          logger.trace("sincedb_write: cleaned", "key" => "'#{key}'")
         end
         @sincedb_last_write = time
         @write_requested = false
       rescue Errno::EACCES
         # no file handles free perhaps
         # maybe it will work next time
-        logger.debug("sincedb_write: error: #{path}: #{$!}")
+        logger.trace("sincedb_write: error: #{path}: #{$!}")
       end
     end
 

data/lib/filewatch/sincedb_value.rb

@@ -66,6 +66,12 @@ module FileWatch
       @watched_file = nil
     end
 
+    def reading_completed
+      touch
+      @path_in_sincedb = @watched_file.path
+      @position = @watched_file.bytes_read
+    end
+
     def unset_watched_file
       # called in read mode only because we flushed any remaining bytes as a final line.
       # cache the position

data/lib/filewatch/stat/generic.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+module FileWatch module Stat
+  class Generic
+
+    attr_reader :identifier, :inode, :modified_at, :size, :inode_struct
+
+    def initialize(source)
+      @source = source
+      @identifier = nil
+      restat
+    end
+
+    def add_identifier(identifier) self; end
+
+    def restat
+      @inner_stat = @source.stat
+      @inode = @inner_stat.ino.to_s
+      @modified_at = @inner_stat.mtime.to_f
+      @size = @inner_stat.size
+      @dev_major = @inner_stat.dev_major
+      @dev_minor = @inner_stat.dev_minor
+      @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor)
+    end
+
+    def windows?
+      false
+    end
+
+    def inspect
+      "<Generic size='#{@size}', modified_at='#{@modified_at}', inode='#{@inode}', inode_struct='#{@inode_struct}'>"
+    end
+  end
+end end

data/lib/filewatch/stat/windows_path.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+module FileWatch module Stat
+  class WindowsPath
+
+    attr_reader :identifier, :inode, :modified_at, :size, :inode_struct
+
+    def initialize(source)
+      @source = source
+      @inode = Winhelper.identifier_from_path(@source.to_path)
+      @dev_major = 0
+      @dev_minor = 0
+      # in windows the dev hi and low are in the identifier
+      @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor)
+      restat
+    end
+
+    def restat
+      @inner_stat = @source.stat
+      @modified_at = @inner_stat.mtime.to_f
+      @size = @inner_stat.size
+    end
+
+    def windows?
+      true
+    end
+
+    def inspect
+      "<WindowsPath size='#{@size}', modified_at='#{@modified_at}', inode='#{@inode}', inode_struct='#{@inode_struct}'>"
+    end
+  end
+end end

data/lib/filewatch/tail_mode/handlers/base.rb

@@ -13,7 +13,7 @@ module FileWatch module TailMode module Handlers
     end
 
     def handle(watched_file)
-      logger.debug("handling: #{watched_file.path}")
+      logger.trace("handling: #{watched_file.filename}")
       unless watched_file.has_listener?
         watched_file.set_listener(@observer)
       end
@@ -30,8 +30,9 @@ module FileWatch module TailMode module Handlers
 
     private
 
-    def read_to_eof(watched_file)
+    def controlled_read(watched_file, loop_control)
       changed = false
+      logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename)
       # from a real config (has 102 file inputs)
       # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously
       # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF
@@ -39,20 +40,16 @@ module FileWatch module TailMode module Handlers
       # we enable the pseudo parallel processing of each file.
       # user also has the option to specify a low `stat_interval` and a very high `discover_interval`to respond
       # quicker to changing files and not allowing too much content to build up before reading it.
-      @settings.file_chunk_count.times do
+      loop_control.count.times do
         begin
-          data = watched_file.file_read(@settings.file_chunk_size)
-          result = watched_file.buffer_extract(data) # expect BufferExtractResult
-          logger.info(result.warning, result.additional) unless result.warning.empty?
+          result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult
+          logger.trace(result.warning, result.additional) unless result.warning.empty?
           changed = true
           result.lines.each do |line|
             watched_file.listener.accept(line)
             # sincedb position is now independent from the watched_file bytes_read
             sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
           end
-          # instead of tracking the bytes_read line by line we need to track by the data read size.
-          # because we seek to the bytes_read not the sincedb position
-          watched_file.increment_bytes_read(data.bytesize)
         rescue EOFError
           # it only makes sense to signal EOF in "read" mode not "tail"
           break
@@ -70,7 +67,7 @@ module FileWatch module TailMode module Handlers
 
     def open_file(watched_file)
       return true if watched_file.file_open?
-      logger.debug("opening #{watched_file.path}")
+      logger.trace("opening #{watched_file.filename}")
       begin
         watched_file.open
       rescue
@@ -82,43 +79,64 @@ module FileWatch module TailMode module Handlers
           logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
           watched_file.last_open_warning_at = now
         else
-          logger.debug("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
+          logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
         end
         watched_file.watch # set it back to watch so we can try it again
-      end
-      if watched_file.file_open?
-        watched_file.listener.opened
-        true
       else
-        false
+        watched_file.listener.opened
       end
+      watched_file.file_open?
     end
 
     def add_or_update_sincedb_collection(watched_file)
       sincedb_value = @sincedb_collection.find(watched_file)
       if sincedb_value.nil?
-        add_new_value_sincedb_collection(watched_file)
+        sincedb_value = add_new_value_sincedb_collection(watched_file)
+        watched_file.initial_completed
       elsif sincedb_value.watched_file == watched_file
         update_existing_sincedb_collection_value(watched_file, sincedb_value)
+        watched_file.initial_completed
       else
-        logger.warn? && logger.warn("mismatch on sincedb_value.watched_file, this should have been handled by Discoverer")
+        msg = "add_or_update_sincedb_collection: found sincedb record"
+        logger.trace(msg,
+          "sincedb key" => watched_file.sincedb_key,
+          "sincedb value" => sincedb_value
+        )
+        # detected a rotation, Discoverer can't handle this because this watched file is not a new discovery.
+        # we must handle it here, by transferring state and have the sincedb value track this watched file
+        # rotate_as_file and rotate_from will switch the sincedb key to the inode that the path is now pointing to
+        # and pickup the sincedb_value from before.
+        msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
+        logger.trace(msg)
+        existing_watched_file = sincedb_value.watched_file
+        if existing_watched_file.nil?
+          sincedb_value.set_watched_file(watched_file)
+          logger.trace("add_or_update_sincedb_collection: switching as new file")
+          watched_file.rotate_as_file
+          watched_file.update_bytes_read(sincedb_value.position)
+        else
+          sincedb_value.set_watched_file(watched_file)
+          logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details)
+          watched_file.rotate_from(existing_watched_file)
+        end
       end
-      watched_file.initial_completed
+      sincedb_value
     end
 
     def update_existing_sincedb_collection_value(watched_file, sincedb_value)
-      logger.debug("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
+      logger.trace("update_existing_sincedb_collection_value: #{watched_file.filename}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
       update_existing_specifically(watched_file, sincedb_value)
     end
 
     def add_new_value_sincedb_collection(watched_file)
       sincedb_value = get_new_value_specifically(watched_file)
-      logger.debug("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position)
+      logger.trace("add_new_value_sincedb_collection", "position" => sincedb_value.position, "watched_file details" => watched_file.details)
       sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
+      sincedb_value
     end
 
     def get_new_value_specifically(watched_file)
-      position = @settings.start_new_files_at == :beginning ? 0 : watched_file.last_stat_size
+      position = watched_file.position_for_new_sincedb_value
       value = SincedbValue.new(position)
       value.set_watched_file(watched_file)
       watched_file.update_bytes_read(position)