logstash-input-file 4.1.3 → 4.1.4

data/lib/filewatch/tail_mode/handlers/create.rb

@@ -10,8 +10,7 @@ module FileWatch module TailMode module Handlers
 
     def update_existing_specifically(watched_file, sincedb_value)
       # sincedb_value is the source of truth
-      position = sincedb_value.position
-      watched_file.update_bytes_read(position)
+      watched_file.update_bytes_read(sincedb_value.position)
     end
   end
 end end end

data/lib/filewatch/tail_mode/handlers/create_initial.rb

@@ -4,6 +4,7 @@ module FileWatch module TailMode module Handlers
   class CreateInitial < Base
     def handle_specifically(watched_file)
       if open_file(watched_file)
+        logger.trace("handle_specifically opened file handle: #{watched_file.file.fileno}, path: #{watched_file.filename}")
         add_or_update_sincedb_collection(watched_file)
       end
     end
@@ -13,7 +14,7 @@ module FileWatch module TailMode module Handlers
       if @settings.start_new_files_at == :beginning
         position = 0
       end
-      logger.debug("update_existing_specifically - #{watched_file.path}: seeking to #{position}")
+      logger.trace("update_existing_specifically - #{watched_file.path}: seeking to #{position}")
       watched_file.update_bytes_read(position)
       sincedb_value.update_position(position)
     end

data/lib/filewatch/tail_mode/handlers/delete.rb

@@ -2,9 +2,21 @@
 
 module FileWatch module TailMode module Handlers
   class Delete < Base
+    DATA_LOSS_WARNING = "watched file path was deleted or rotated before all content was read, if the file is found again it will be read from the last position"
     def handle_specifically(watched_file)
+      # TODO consider trying to find the renamed file - it will have the same inode.
+      # Needs a rotate scheme rename hint from user e.g. "<name>-YYYY-MM-DD-N.<ext>" or "<name>.<ext>.N"
+      # send the found content to the same listener (stream identity)
+      logger.trace("info",
+        "watched_file details" => watched_file.details,
+        "path" => watched_file.path)
+      if watched_file.bytes_unread > 0
+        logger.warn(DATA_LOSS_WARNING, "unread_bytes" => watched_file.bytes_unread, "path" => watched_file.path)
+      end
       watched_file.listener.deleted
-      sincedb_collection.unset_watched_file(watched_file)
+      # no need to worry about data in the buffer
+      # if found it will be associated by inode and read from last position
+      sincedb_collection.watched_file_deleted(watched_file)
       watched_file.file_close
     end
   end

data/lib/filewatch/tail_mode/handlers/grow.rb

@@ -4,8 +4,11 @@ module FileWatch module TailMode module Handlers
   class Grow < Base
     def handle_specifically(watched_file)
       watched_file.file_seek(watched_file.bytes_read)
-      logger.debug("reading to eof: #{watched_file.path}")
-      read_to_eof(watched_file)
+      loop do
+        loop_control = watched_file.loop_control_adjusted_for_stat_size
+        controlled_read(watched_file, loop_control)
+        break unless loop_control.more
+      end
     end
   end
 end end end

data/lib/filewatch/tail_mode/handlers/shrink.rb

@@ -5,16 +5,19 @@ module FileWatch module TailMode module Handlers
     def handle_specifically(watched_file)
       add_or_update_sincedb_collection(watched_file)
       watched_file.file_seek(watched_file.bytes_read)
-      logger.debug("reading to eof: #{watched_file.path}")
-      read_to_eof(watched_file)
+      loop do
+        loop_control = watched_file.loop_control_adjusted_for_stat_size
+        controlled_read(watched_file, loop_control)
+        break unless loop_control.more
+      end
     end
 
     def update_existing_specifically(watched_file, sincedb_value)
       # we have a match but size is smaller
       # set all to zero
-      logger.debug("update_existing_specifically: #{watched_file.path}: was truncated seeking to beginning")
-      watched_file.update_bytes_read(0) if watched_file.bytes_read != 0
+      watched_file.reset_bytes_unread
       sincedb_value.update_position(0)
+      logger.trace("update_existing_specifically: was truncated seeking to beginning", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
     end
   end
 end end end

data/lib/filewatch/tail_mode/handlers/unignore.rb

@@ -6,15 +6,16 @@ module FileWatch module TailMode module Handlers
     # before any other handling has been done
     # at a minimum we create or associate a sincedb value
     def handle_specifically(watched_file)
-      add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
+      add_or_update_sincedb_collection(watched_file)
     end
 
     def get_new_value_specifically(watched_file)
       # for file initially ignored their bytes_read was set to stat.size
       # use this value not the `start_new_files_at` for the position
-      # logger.debug("get_new_value_specifically", "watched_file" => watched_file.inspect)
+      # logger.trace("get_new_value_specifically", "watched_file" => watched_file.inspect)
       SincedbValue.new(watched_file.bytes_read).tap do |val|
         val.set_watched_file(watched_file)
+        logger.trace("-------------------- >>>>> get_new_value_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => val)
       end
     end
 
@@ -25,6 +26,7 @@ module FileWatch module TailMode module Handlers
       # we will handle grow or shrink
       # for now we seek to where we were before the file got ignored (grow)
       # or to the start (shrink)
+      logger.trace("-------------------- >>>>> update_existing_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
       position = 0
       if watched_file.shrunk?
         watched_file.update_bytes_read(0)

data/lib/filewatch/tail_mode/processor.rb

@@ -34,6 +34,7 @@ module FileWatch module TailMode
     end
 
     def initialize_handlers(sincedb_collection, observer)
+      @sincedb_collection = sincedb_collection
       @create_initial = Handlers::CreateInitial.new(sincedb_collection, observer, @settings)
       @create = Handlers::Create.new(sincedb_collection, observer, @settings)
       @grow = Handlers::Grow.new(sincedb_collection, observer, @settings)
@@ -71,80 +72,148 @@ module FileWatch module TailMode
       @unignore.handle(watched_file)
     end
 
+    def process_all_states(watched_files)
+      process_closed(watched_files)
+      return if watch.quit?
+      process_ignored(watched_files)
+      return if watch.quit?
+      process_delayed_delete(watched_files)
+      return if watch.quit?
+      process_restat_for_watched_and_active(watched_files)
+      return if watch.quit?
+      process_rotation_in_progress(watched_files)
+      return if watch.quit?
+      process_watched(watched_files)
+      return if watch.quit?
+      process_active(watched_files)
+    end
+
+    private
+
     def process_closed(watched_files)
-      logger.debug("Closed processing")
+      # logger.trace("Closed processing")
       # Handles watched_files in the closed state.
       # if its size changed it is put into the watched state
       watched_files.select {|wf| wf.closed? }.each do |watched_file|
-        path = watched_file.path
-        begin
-          watched_file.restat
+        common_restat_with_delay(watched_file, "Closed") do
+          # it won't do this if rotation is detected
           if watched_file.size_changed?
             # if the closed file changed, move it to the watched state
             # not to active state because we want to respect the active files window.
             watched_file.watch
           end
-        rescue Errno::ENOENT
-          # file has gone away or we can't read it anymore.
-          common_deleted_reaction(watched_file, "Closed")
-        rescue => e
-          common_error_reaction(path, e, "Closed")
         end
         break if watch.quit?
       end
     end
 
     def process_ignored(watched_files)
-      logger.debug("Ignored processing")
+      # logger.trace("Ignored processing")
       # Handles watched_files in the ignored state.
       # if its size changed:
       #   put it in the watched state
       #   invoke unignore
       watched_files.select {|wf| wf.ignored? }.each do |watched_file|
-        path = watched_file.path
-        begin
-          watched_file.restat
+        common_restat_with_delay(watched_file, "Ignored") do
+          # it won't do this if rotation is detected
           if watched_file.size_changed?
             watched_file.watch
             unignore(watched_file)
           end
-        rescue Errno::ENOENT
-          # file has gone away or we can't read it anymore.
-          common_deleted_reaction(watched_file, "Ignored")
-        rescue => e
-          common_error_reaction(path, e, "Ignored")
         end
         break if watch.quit?
       end
     end
 
+    def process_delayed_delete(watched_files)
+      # defer the delete to one loop later to ensure that the stat really really can't find a renamed file
+      # because a `stat` can be called right in the middle of the rotation rename cascade
+      logger.trace("Delayed Delete processing")
+      watched_files.select {|wf| wf.delayed_delete?}.each do |watched_file|
+        logger.trace(">>> Delayed Delete", "path" => watched_file.filename)
+        common_restat_without_delay(watched_file, ">>> Delayed Delete") do
+          logger.trace(">>> Delayed Delete: file at path found again", "watched_file" => watched_file.details)
+          watched_file.file_at_path_found_again
+        end
+      end
+    end
+
+    def process_restat_for_watched_and_active(watched_files)
+      # do restat on all watched and active states once now. closed and ignored have been handled already
+      logger.trace("Watched + Active restat processing")
+      watched_files.select {|wf| wf.watched? || wf.active?}.each do |watched_file|
+        common_restat_with_delay(watched_file, "Watched")
+      end
+    end
+
+    def process_rotation_in_progress(watched_files)
+      logger.trace("Rotation In Progress processing")
+      watched_files.select {|wf| wf.rotation_in_progress?}.each do |watched_file|
+        if !watched_file.all_read?
+          if watched_file.file_open?
+            # rotated file but original opened file is not fully read
+            # we need to keep reading the open file, if we close it we lose it because the path is now pointing at a different file.
+            logger.trace(">>> Rotation In Progress - inode change detected and original content is not fully read, reading all", "watched_file details" => watched_file.details)
+            # need to fully read open file while we can
+            watched_file.set_maximum_read_loop
+            grow(watched_file)
+            watched_file.set_standard_read_loop
+          else
+            logger.warn(">>> Rotation In Progress - inode change detected and original content is not fully read, file is closed and path points to new content", "watched_file details" => watched_file.details)
+          end
+        end
+        current_key = watched_file.sincedb_key
+        sdb_value = @sincedb_collection.get(current_key)
+        potential_key = watched_file.stat_sincedb_key
+        potential_sdb_value =  @sincedb_collection.get(potential_key)
+        logger.trace(">>> Rotation In Progress", "watched_file" => watched_file.details, "found_sdb_value" => sdb_value, "potential_key" => potential_key, "potential_sdb_value" => potential_sdb_value)
+        if potential_sdb_value.nil?
+          if sdb_value.nil?
+            logger.trace("---------- >>> Rotation In Progress: rotating as initial file, no potential sincedb value AND no found sincedb value")
+            watched_file.rotate_as_initial_file
+          else
+            logger.trace("---------- >>>> Rotation In Progress: rotating as existing file, no potential sincedb value BUT found sincedb value")
+            watched_file.rotate_as_file
+            sdb_value.clear_watched_file
+          end
+          new_sdb_value = SincedbValue.new(0)
+          new_sdb_value.set_watched_file(watched_file)
+          @sincedb_collection.set(potential_key, new_sdb_value)
+        else
+          other_watched_file = potential_sdb_value.watched_file
+          if other_watched_file.nil?
+            logger.trace("---------- >>>> Rotation In Progress: rotating as existing file WITH potential sincedb value that does not have a watched file reference !!!!!!!!!!!!!!!!!")
+            watched_file.rotate_as_file(potential_sdb_value.position)
+            sdb_value.clear_watched_file unless sdb_value.nil?
+            potential_sdb_value.set_watched_file(watched_file)
+          else
+            logger.trace("---------- >>>> Rotation In Progress: rotating from...", "this watched_file details" => watched_file.details, "other watched_file details" => other_watched_file.details)
+            watched_file.rotate_from(other_watched_file)
+            sdb_value.clear_watched_file unless sdb_value.nil?
+            potential_sdb_value.set_watched_file(watched_file)
+          end
+        end
+        logger.trace("---------- >>>> Rotation In Progress: after handling rotation", "this watched_file details" => watched_file.details, "sincedb_value" => (potential_sdb_value || sdb_value))
+      end
+    end
+
     def process_watched(watched_files)
-      logger.debug("Watched processing")
       # Handles watched_files in the watched state.
       # for a slice of them:
       #   move to the active state
       #   and we allow the block to open the file and create a sincedb collection record if needed
       #   some have never been active and some have
       #   those that were active before but are watched now were closed under constraint
-
+      logger.trace("Watched processing")
       # how much of the max active window is available
       to_take = @settings.max_active - watched_files.count{|wf| wf.active?}
       if to_take > 0
         watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file|
-          path = watched_file.path
-          begin
-            watched_file.restat
-            watched_file.activate
-            if watched_file.initial?
-              create_initial(watched_file)
-            else
-              create(watched_file)
-            end
-          rescue Errno::ENOENT
-            # file has gone away or we can't read it anymore.
-            common_deleted_reaction(watched_file, "Watched")
-          rescue => e
-            common_error_reaction(path, e, "Watched")
+          watched_file.activate
+          if watched_file.initial?
+            create_initial(watched_file)
+          else
+            create(watched_file)
           end
           break if watch.quit?
         end
@@ -159,51 +228,71 @@ module FileWatch module TailMode
     end
 
     def process_active(watched_files)
-      logger.debug("Active processing")
+      # logger.trace("Active processing")
       # Handles watched_files in the active state.
-      # it has been read once - unless they were empty at the time
+      # files have been opened at this point
       watched_files.select {|wf| wf.active? }.each do |watched_file|
-        path = watched_file.path
-        begin
-          watched_file.restat
-        rescue Errno::ENOENT
-          # file has gone away or we can't read it anymore.
-          common_deleted_reaction(watched_file, "Active")
-          next
-        rescue => e
-          common_error_reaction(path, e, "Active")
-          next
-        end
         break if watch.quit?
+        path = watched_file.filename
         if watched_file.grown?
-          logger.debug("Active - file grew: #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}")
+          logger.trace("Active - file grew: #{path}: new size is #{watched_file.last_stat_size}, bytes read #{watched_file.bytes_read}")
           grow(watched_file)
         elsif watched_file.shrunk?
+          if watched_file.bytes_unread > 0
+            logger.warn("Active - shrunk: DATA LOSS!! truncate detected with #{watched_file.bytes_unread} unread bytes: #{path}")
+          end
           # we don't update the size here, its updated when we actually read
-          logger.debug("Active - file shrunk #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}")
+          logger.trace("Active - file shrunk #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}")
           shrink(watched_file)
         else
           # same size, do nothing
+          logger.trace("Active - no change", "watched_file" => watched_file.details)
         end
         # can any active files be closed to make way for waiting files?
         if watched_file.file_closable?
-          logger.debug("Watch each: active: file expired: #{path}")
+          logger.trace("Watch each: active: file expired: #{path}")
           timeout(watched_file)
           watched_file.close
         end
       end
     end
 
-    def common_deleted_reaction(watched_file, action)
-      # file has gone away or we can't read it anymore.
-      watched_file.unwatch
-      delete(watched_file)
-      deletable_filepaths << watched_file.path
-      logger.debug("#{action} - stat failed: #{watched_file.path}, removing from collection")
+    def common_restat_with_delay(watched_file, action, &block)
+      common_restat(watched_file, action, true, &block)
     end
 
-    def common_error_reaction(path, error, action)
-      logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})")
+    def common_restat_without_delay(watched_file, action, &block)
+      common_restat(watched_file, action, false, &block)
+    end
+
+    def common_restat(watched_file, action, delay, &block)
+      all_ok = true
+      begin
+        watched_file.restat
+        if watched_file.rotation_in_progress?
+          logger.trace("-------------------- >>>>> restat - rotation_detected", "watched_file details" => watched_file.details, "new sincedb key" => watched_file.stat_sincedb_key)
+          # don't yield to closed and ignore processing
+        else
+          yield if block_given?
+        end
+      rescue Errno::ENOENT
+        if delay
+          logger.trace("#{action} - delaying the stat fail on: #{watched_file.filename}")
+          watched_file.delay_delete
+        else
+          # file has gone away or we can't read it anymore.
+          logger.trace("#{action} - after a delay, really can't find this file: #{watched_file.filename}")
+          watched_file.unwatch
+          logger.trace("#{action} - removing from collection: #{watched_file.filename}")
+          delete(watched_file)
+          deletable_filepaths << watched_file.path
+          all_ok = false
+        end
+      rescue => e
+        logger.error("#{action} - other error #{watched_file.path}: (#{e.message}, #{e.backtrace.take(8).inspect})")
+        all_ok = false
+      end
+      all_ok
     end
   end
 end end

data/lib/filewatch/watch.rb

@@ -10,11 +10,8 @@ module FileWatch
 
     def initialize(discoverer, watched_files_collection, settings)
       @settings = settings
-      # watch and iterate_on_state can be called from different threads.
-      @lock = Mutex.new
       # we need to be threadsafe about the quit mutation
-      @quit = false
-      @quit_lock = Mutex.new
+      @quit = Concurrent::AtomicBoolean.new(false)
       @lastwarn_max_files = 0
       @discoverer = discoverer
       @watched_files_collection = watched_files_collection
@@ -27,17 +24,13 @@ module FileWatch
     end
 
     def watch(path)
-      synchronized do
-        @discoverer.add_path(path)
-      end
+      @discoverer.add_path(path)
       # don't return whatever @discoverer.add_path returns
       return true
     end
 
     def discover
-      synchronized do
-        @discoverer.discover
-      end
+      @discoverer.discover
       # don't return whatever @discoverer.discover returns
       return true
     end
@@ -60,6 +53,7 @@ module FileWatch
         break if quit?
         sleep(@settings.stat_interval)
       end
+      sincedb_collection.write_if_requested # does nothing if no requests to write were lodged.
       @watched_files_collection.close_all
     end # def subscribe
 
@@ -67,42 +61,28 @@ module FileWatch
     # differently from Tail mode - see the ReadMode::Processor and TailMode::Processor
     def iterate_on_state
       return if @watched_files_collection.empty?
-      synchronized do
-        begin
-          # creates this snapshot of watched_file values just once
-          watched_files = @watched_files_collection.values
-          @processor.process_closed(watched_files)
-          return if quit?
-          @processor.process_ignored(watched_files)
-          return if quit?
-          @processor.process_watched(watched_files)
-          return if quit?
-          @processor.process_active(watched_files)
-        ensure
-          @watched_files_collection.delete(@processor.deletable_filepaths)
-          @processor.deletable_filepaths.clear
-        end
+      begin
+        # creates this snapshot of watched_file values just once
+        watched_files = @watched_files_collection.values
+        @processor.process_all_states(watched_files)
+      ensure
+        @watched_files_collection.delete(@processor.deletable_filepaths)
+        @processor.deletable_filepaths.clear
       end
     end # def each
 
     def quit
-      @quit_lock.synchronize do
-        @quit = true
-      end
-    end # def quit
+      @quit.make_true
+    end
 
     def quit?
-      @quit_lock.synchronize { @quit }
+      @quit.true?
     end
 
     private
 
-    def synchronized(&block)
-      @lock.synchronize { block.call }
-    end
-
     def reset_quit
-      @quit_lock.synchronize { @quit = false }
+      @quit.make_false
     end
   end
 end