logstash-input-file 4.1.18 → 4.2.4

data/lib/filewatch/tail_mode/handlers/delete.rb

@@ -7,11 +7,9 @@ module FileWatch module TailMode module Handlers
       # TODO consider trying to find the renamed file - it will have the same inode.
       # Needs a rotate scheme rename hint from user e.g. "<name>-YYYY-MM-DD-N.<ext>" or "<name>.<ext>.N"
       # send the found content to the same listener (stream identity)
-      logger.trace("info",
-        "watched_file details" => watched_file.details,
-        "path" => watched_file.path)
+      logger.trace? && logger.trace(__method__.to_s, :path => watched_file.path, :watched_file => watched_file.details)
       if watched_file.bytes_unread > 0
-        logger.warn(DATA_LOSS_WARNING, "unread_bytes" => watched_file.bytes_unread, "path" => watched_file.path)
+        logger.warn(DATA_LOSS_WARNING, :path => watched_file.path, :unread_bytes => watched_file.bytes_unread)
       end
       watched_file.listener.deleted
       # no need to worry about data in the buffer

data/lib/filewatch/tail_mode/handlers/shrink.rb

@@ -14,11 +14,10 @@ module FileWatch module TailMode module Handlers
     end
 
     def update_existing_specifically(watched_file, sincedb_value)
-      # we have a match but size is smaller
-      # set all to zero
+      # we have a match but size is smaller - set all to zero
       watched_file.reset_bytes_unread
       sincedb_value.update_position(0)
-      logger.trace("update_existing_specifically: was truncated seeking to beginning", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
+      logger.trace? && logger.trace("update_existing_specifically: was truncated seeking to beginning", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
     end
   end
 end end end

data/lib/filewatch/tail_mode/handlers/unignore.rb

@@ -13,9 +13,9 @@ module FileWatch module TailMode module Handlers
       # for file initially ignored their bytes_read was set to stat.size
       # use this value not the `start_new_files_at` for the position
       # logger.trace("get_new_value_specifically", "watched_file" => watched_file.inspect)
-      SincedbValue.new(watched_file.bytes_read).tap do |val|
-        val.set_watched_file(watched_file)
-        logger.trace("-------------------- >>>>> get_new_value_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => val)
+      SincedbValue.new(watched_file.bytes_read).tap do |sincedb_value|
+        sincedb_value.set_watched_file(watched_file)
+        logger.trace? && logger.trace("get_new_value_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
       end
     end
 
@@ -26,7 +26,7 @@ module FileWatch module TailMode module Handlers
       # we will handle grow or shrink
       # for now we seek to where we were before the file got ignored (grow)
       # or to the start (shrink)
-      logger.trace("-------------------- >>>>> update_existing_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
+      logger.trace? && logger.trace("update_existing_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
       position = 0
       if watched_file.shrunk?
         watched_file.update_bytes_read(0)

data/lib/filewatch/tail_mode/processor.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require "logstash/util/loggable"
+require 'filewatch/processor'
 require_relative "handlers/base"
 require_relative "handlers/create_initial"
 require_relative "handlers/create"
@@ -18,20 +18,7 @@ module FileWatch module TailMode
   #   :delete   - file can't be read
   #   :timeout - file is closable
   #   :unignore - file was ignored, but have now received new content
-  class Processor
-    include LogStash::Util::Loggable
-
-    attr_reader :watch, :deletable_filepaths
-
-    def initialize(settings)
-      @settings = settings
-      @deletable_filepaths = []
-    end
-
-    def add_watch(watch)
-      @watch = watch
-      self
-    end
+  class Processor < FileWatch::Processor
 
     def initialize_handlers(sincedb_collection, observer)
       @sincedb_collection = sincedb_collection
@@ -91,11 +78,12 @@ module FileWatch module TailMode
     private
 
     def process_closed(watched_files)
-      # logger.trace("Closed processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the closed state.
       # if its size changed it is put into the watched state
-      watched_files.select {|wf| wf.closed? }.each do |watched_file|
-        common_restat_with_delay(watched_file, "Closed") do
+      watched_files.each do |watched_file|
+        next unless watched_file.closed?
+        common_restat_with_delay(watched_file, __method__) do
           # it won't do this if rotation is detected
           if watched_file.size_changed?
             # if the closed file changed, move it to the watched state
@@ -108,13 +96,14 @@ module FileWatch module TailMode
     end
 
     def process_ignored(watched_files)
-      # logger.trace("Ignored processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the ignored state.
       # if its size changed:
       #   put it in the watched state
       #   invoke unignore
-      watched_files.select {|wf| wf.ignored? }.each do |watched_file|
-        common_restat_with_delay(watched_file, "Ignored") do
+      watched_files.each do |watched_file|
+        next unless watched_file.ignored?
+        common_restat_with_delay(watched_file, __method__) do
           # it won't do this if rotation is detected
           if watched_file.size_changed?
             watched_file.watch
@@ -128,11 +117,12 @@ module FileWatch module TailMode
     def process_delayed_delete(watched_files)
       # defer the delete to one loop later to ensure that the stat really really can't find a renamed file
       # because a `stat` can be called right in the middle of the rotation rename cascade
-      logger.trace("Delayed Delete processing")
-      watched_files.select {|wf| wf.delayed_delete?}.each do |watched_file|
-        logger.trace(">>> Delayed Delete", "path" => watched_file.filename)
-        common_restat_without_delay(watched_file, ">>> Delayed Delete") do
-          logger.trace(">>> Delayed Delete: file at path found again", "watched_file" => watched_file.details)
+      logger.trace(__method__.to_s)
+      watched_files.each do |watched_file|
+        next unless watched_file.delayed_delete?
+        logger.trace(">>> Delayed Delete", :path => watched_file.path)
+        common_restat_without_delay(watched_file, __method__) do
+          logger.trace(">>> Delayed Delete: file at path found again", :watched_file => watched_file.details)
           watched_file.file_at_path_found_again
         end
       end
@@ -140,33 +130,35 @@ module FileWatch module TailMode
 
     def process_restat_for_watched_and_active(watched_files)
       # do restat on all watched and active states once now. closed and ignored have been handled already
-      logger.trace("Watched + Active restat processing")
-      watched_files.select {|wf| wf.watched? || wf.active?}.each do |watched_file|
-        common_restat_with_delay(watched_file, "Watched")
+      logger.trace(__method__.to_s)
+      watched_files.each do |watched_file|
+        next if !watched_file.watched? && !watched_file.active?
+        common_restat_with_delay(watched_file, __method__)
       end
     end
 
     def process_rotation_in_progress(watched_files)
-      logger.trace("Rotation In Progress processing")
-      watched_files.select {|wf| wf.rotation_in_progress?}.each do |watched_file|
+      logger.trace(__method__.to_s)
+      watched_files.each do |watched_file|
+        next unless watched_file.rotation_in_progress?
         if !watched_file.all_read?
           if watched_file.file_open?
             # rotated file but original opened file is not fully read
             # we need to keep reading the open file, if we close it we lose it because the path is now pointing at a different file.
-            logger.trace(">>> Rotation In Progress - inode change detected and original content is not fully read, reading all", "watched_file details" => watched_file.details)
+            logger.trace(">>> Rotation In Progress - inode change detected and original content is not fully read, reading all", :watched_file => watched_file.details)
             # need to fully read open file while we can
             watched_file.set_maximum_read_loop
             grow(watched_file)
             watched_file.set_standard_read_loop
           else
-            logger.warn(">>> Rotation In Progress - inode change detected and original content is not fully read, file is closed and path points to new content", "watched_file details" => watched_file.details)
+            logger.warn(">>> Rotation In Progress - inode change detected and original content is not fully read, file is closed and path points to new content", :watched_file => watched_file.details)
           end
         end
         current_key = watched_file.sincedb_key
         sdb_value = @sincedb_collection.get(current_key)
         potential_key = watched_file.stat_sincedb_key
         potential_sdb_value =  @sincedb_collection.get(potential_key)
-        logger.trace(">>> Rotation In Progress", "watched_file" => watched_file.details, "found_sdb_value" => sdb_value, "potential_key" => potential_key, "potential_sdb_value" => potential_sdb_value)
+        logger.trace(">>> Rotation In Progress", :watched_file => watched_file.details, :found_sdb_value => sdb_value, :potential_key => potential_key, :potential_sdb_value => potential_sdb_value)
         if potential_sdb_value.nil?
           logger.trace("---------- >>>> Rotation In Progress: rotating as existing file")
           watched_file.rotate_as_file
@@ -189,13 +181,13 @@ module FileWatch module TailMode
             sdb_value.clear_watched_file unless sdb_value.nil?
             potential_sdb_value.set_watched_file(watched_file)
           else
-            logger.trace("---------- >>>> Rotation In Progress: rotating from...", "this watched_file details" => watched_file.details, "other watched_file details" => other_watched_file.details)
+            logger.trace("---------- >>>> Rotation In Progress: rotating from...", :this_watched_file => watched_file.details, :other_watched_file => other_watched_file.details)
             watched_file.rotate_from(other_watched_file)
             sdb_value.clear_watched_file unless sdb_value.nil?
             potential_sdb_value.set_watched_file(watched_file)
           end
         end
-        logger.trace("---------- >>>> Rotation In Progress: after handling rotation", "this watched_file details" => watched_file.details, "sincedb_value" => (potential_sdb_value || sdb_value))
+        logger.trace("---------- >>>> Rotation In Progress: after handling rotation", :this_watched_file => watched_file.details, :sincedb_value => (potential_sdb_value || sdb_value))
       end
     end
 
@@ -206,11 +198,11 @@ module FileWatch module TailMode
       #   and we allow the block to open the file and create a sincedb collection record if needed
       #   some have never been active and some have
       #   those that were active before but are watched now were closed under constraint
-      logger.trace("Watched processing")
+      logger.trace(__method__.to_s)
       # how much of the max active window is available
-      to_take = @settings.max_active - watched_files.count{|wf| wf.active?}
+      to_take = @settings.max_active - watched_files.count(&:active?)
       if to_take > 0
-        watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file|
+        watched_files.select(&:watched?).take(to_take).each do |watched_file|
           watched_file.activate
           if watched_file.initial?
             create_initial(watched_file)
@@ -223,36 +215,37 @@ module FileWatch module TailMode
         now = Time.now.to_i
         if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL
           waiting = watched_files.size - @settings.max_active
-          logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}")
+          logger.warn("#{@settings.max_warn_msg}, files yet to open: #{waiting}")
           watch.lastwarn_max_files = now
         end
       end
     end
 
     def process_active(watched_files)
-      # logger.trace("Active processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the active state.
       # files have been opened at this point
-      watched_files.select {|wf| wf.active? }.each do |watched_file|
+      watched_files.each do |watched_file|
+        next unless watched_file.active?
         break if watch.quit?
         path = watched_file.filename
         if watched_file.grown?
-          logger.trace("Active - file grew: #{path}: new size is #{watched_file.last_stat_size}, bytes read #{watched_file.bytes_read}")
+          logger.trace("#{__method__} file grew: new size is #{watched_file.last_stat_size}, bytes read #{watched_file.bytes_read}", :path => path)
           grow(watched_file)
         elsif watched_file.shrunk?
           if watched_file.bytes_unread > 0
-            logger.warn("Active - shrunk: DATA LOSS!! truncate detected with #{watched_file.bytes_unread} unread bytes: #{path}")
+            logger.warn("potential data loss, file truncate detected with #{watched_file.bytes_unread} unread bytes", :path => path)
           end
           # we don't update the size here, its updated when we actually read
-          logger.trace("Active - file shrunk #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}")
+          logger.trace("#{__method__} file shrunk: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}", :path => path)
           shrink(watched_file)
         else
           # same size, do nothing
-          logger.trace("Active - no change", "watched_file" => watched_file.details)
+          logger.trace("#{__method__} no change", :path => path)
         end
         # can any active files be closed to make way for waiting files?
         if watched_file.file_closable?
-          logger.trace("Watch each: active: file expired: #{path}")
+          logger.trace("#{__method__} file expired", :path => path)
           timeout(watched_file)
           watched_file.close
         end
@@ -270,28 +263,28 @@ module FileWatch module TailMode
     def common_restat(watched_file, action, delay, &block)
       all_ok = true
       begin
-        watched_file.restat
+        restat(watched_file)
         if watched_file.rotation_in_progress?
-          logger.trace("-------------------- >>>>> restat - rotation_detected", "watched_file details" => watched_file.details, "new sincedb key" => watched_file.stat_sincedb_key)
+          logger.trace("-------------------- >>>>> restat - rotation_detected", :watched_file => watched_file.details, :new_sincedb_key => watched_file.stat_sincedb_key)
           # don't yield to closed and ignore processing
         else
           yield if block_given?
         end
       rescue Errno::ENOENT
         if delay
-          logger.trace("#{action} - delaying the stat fail on: #{watched_file.filename}")
+          logger.trace("#{action} - delaying the stat fail on", :filename => watched_file.filename)
           watched_file.delay_delete
         else
           # file has gone away or we can't read it anymore.
-          logger.trace("#{action} - after a delay, really can't find this file: #{watched_file.filename}")
+          logger.trace("#{action} - after a delay, really can't find this file", :path => watched_file.path)
           watched_file.unwatch
-          logger.trace("#{action} - removing from collection: #{watched_file.filename}")
+          logger.trace("#{action} - removing from collection", :filename => watched_file.filename)
           delete(watched_file)
-          deletable_filepaths << watched_file.path
+          add_deletable_path watched_file.path
           all_ok = false
         end
       rescue => e
-        logger.error("#{action} - other error #{watched_file.path}: (#{e.message}, #{e.backtrace.take(8).inspect})")
+        logger.error("#{action} - other error", error_details(e, watched_file))
         all_ok = false
       end
       all_ok

data/lib/filewatch/watch.rb

@@ -1,26 +1,25 @@
 # encoding: utf-8
 require "logstash/util/loggable"
+require "concurrent/atomic/atomic_boolean"
 
 module FileWatch
   class Watch
     include LogStash::Util::Loggable
 
     attr_accessor :lastwarn_max_files
-    attr_reader :discoverer, :watched_files_collection
+    attr_reader :discoverer, :processor, :watched_files_collection
 
-    def initialize(discoverer, watched_files_collection, settings)
+    def initialize(discoverer, processor, settings)
+      @discoverer = discoverer
+      @watched_files_collection = discoverer.watched_files_collection
       @settings = settings
+
       # we need to be threadsafe about the quit mutation
       @quit = Concurrent::AtomicBoolean.new(false)
       @lastwarn_max_files = 0
-      @discoverer = discoverer
-      @watched_files_collection = watched_files_collection
-    end
 
-    def add_processor(processor)
       @processor = processor
       @processor.add_watch(self)
-      self
     end
 
     def watch(path)
@@ -52,7 +51,10 @@ module FileWatch
           glob = 0
         end
         break if quit?
+        # NOTE: maybe the plugin should validate stat_interval <= sincedb_write_interval <= sincedb_clean_after
         sleep(@settings.stat_interval)
+        # we need to check potential expired keys (sincedb_clean_after) periodically
+        sincedb_collection.flush_at_interval
       end
       sincedb_collection.write_if_requested # does nothing if no requests to write were lodged.
       @watched_files_collection.close_all
@@ -67,20 +69,16 @@ module FileWatch
         watched_files = @watched_files_collection.values
         @processor.process_all_states(watched_files)
       ensure
-        @watched_files_collection.remove_paths(@processor.deletable_filepaths)
-        @processor.deletable_filepaths.clear
+        @watched_files_collection.remove_paths(@processor.clear_deletable_paths)
       end
-    end # def each
+    end
 
     def quit
       @quit.make_true
     end
 
     def quit?
-      if @settings.exit_after_read
-        @exit = @watched_files_collection.empty?
-      end
-      @quit.true? || @exit
+      @quit.true? || (@settings.exit_after_read && @watched_files_collection.empty?)
     end
 
     private

data/lib/filewatch/watched_file.rb

@@ -6,7 +6,7 @@ module FileWatch
     IO_BASED_STAT = 1
 
     attr_reader :bytes_read, :state, :file, :buffer, :recent_states, :bytes_unread
-    attr_reader :path, :accessed_at, :modified_at, :pathname, :filename
+    attr_reader :path, :accessed_at, :pathname, :filename
     attr_reader :listener, :read_loop_count, :read_chunk_size, :stat
     attr_reader :loop_count_type, :loop_count_mode
     attr_accessor :last_open_warning_at
@@ -16,7 +16,7 @@ module FileWatch
     def initialize(pathname, stat, settings)
       @settings = settings
       @pathname = Pathname.new(pathname) # given arg pathname might be a string or a Pathname object
-      @path = @pathname.to_path
+      @path = @pathname.to_path.freeze
       @filename = @pathname.basename.to_s
       full_state_reset(stat)
       watch
@@ -24,10 +24,6 @@ module FileWatch
       set_accessed_at
     end
 
-    def no_restat_reset
-      full_state_reset(@stat)
-    end
-
     def full_state_reset(this_stat = nil)
       if this_stat.nil?
         begin
@@ -75,6 +71,7 @@ module FileWatch
       @size = @stat.size
       @sdb_key_v1 = @stat.inode_struct
     end
+    private :set_stat
 
     def rotate_as_file(bytes_read = 0)
       # rotation, when a sincedb record exists for new inode, but no watched file to rotate from
@@ -100,19 +97,33 @@ module FileWatch
       stat_sincedb_key != sincedb_key
     end
 
-    def restat
+    # @return true if the file was modified since last stat
+    def restat!
+      modified_at # to always be able to detect changes
       @stat.restat
       if rotation_detected?
         # switch to new state now
         rotation_in_progress
+        return true
       else
         @size = @stat.size
         update_bytes_unread
+        modified_at_changed?
+      end
+    end
+
+    def modified_at(update = false)
+      if update || @modified_at.nil?
+        @modified_at = @stat.modified_at
+      else
+        @modified_at
       end
     end
 
-    def modified_at
-      @stat.modified_at
+    # @return whether modified_at changed since it was last read
+    # @see #restat!
+    def modified_at_changed?
+      modified_at != @stat.modified_at
     end
 
     def position_for_new_sincedb_value
@@ -405,14 +416,14 @@ module FileWatch
     end
 
     def details
-      detail = "@filename='#{filename}', @state='#{state}', @recent_states='#{@recent_states.inspect}', "
-      detail.concat("@bytes_read='#{@bytes_read}', @bytes_unread='#{@bytes_unread}', current_size='#{current_size}', ")
-      detail.concat("last_stat_size='#{last_stat_size}', file_open?='#{file_open?}', @initial=#{@initial}")
-      "<FileWatch::WatchedFile: #{detail}, @sincedb_key='#{sincedb_key}'>"
+      detail = "@filename='#{@filename}', @state=#{@state.inspect}, @recent_states=#{@recent_states.inspect}, "
+      detail.concat("@bytes_read=#{@bytes_read}, @bytes_unread=#{@bytes_unread}, current_size=#{current_size}, ")
+      detail.concat("last_stat_size=#{last_stat_size}, file_open?=#{file_open?}, @initial=#{@initial}")
+      "<FileWatch::WatchedFile: #{detail}, sincedb_key='#{sincedb_key}'>"
     end
 
     def inspect
-      "\"<FileWatch::WatchedFile: @filename='#{filename}', @state='#{state}', @sincedb_key='#{sincedb_key}, size=#{@size}>\""
+      "<FileWatch::WatchedFile: @filename='#{@filename}', @state=#{@state.inspect}, current_size=#{current_size}, sincedb_key='#{sincedb_key}'>"
     end
 
     def to_s

data/lib/filewatch/watched_files_collection.rb

@@ -1,89 +1,22 @@
 # encoding: utf-8
-module FileWatch
-  class WatchedFilesCollection
-
-    def initialize(settings)
-      @sort_by = settings.file_sort_by # "last_modified" | "path"
-      @sort_direction = settings.file_sort_direction # "asc" | "desc"
-      @sort_method = method("#{@sort_by}_#{@sort_direction}".to_sym)
-      @files = Concurrent::Array.new
-      @pointers = Concurrent::Hash.new
-    end
 
-    def add(watched_file)
-      @files << watched_file
-      @sort_method.call
-    end
+require 'java'
 
-    def remove_paths(paths)
-      removed_files = Array(paths).map do |path|
-        index = @pointers.delete(path)
-        if index
-          watched_file = @files.delete_at(index)
-          refresh_pointers
-          watched_file
-        end
-      end
-      @sort_method.call
-      removed_files
-    end
+module FileWatch
+  # @see `org.logstash.filewatch.WatchedFilesCollection`
+  class WatchedFilesCollection
 
+    # Closes all managed watched files.
+    # @see FileWatch::WatchedFile#file_close
     def close_all
-      @files.each(&:file_close)
-    end
-
-    def empty?
-      @files.empty?
-    end
-
-    def keys
-      @pointers.keys
+      each_file(&:file_close) # synchronized
     end
 
-    def values
-      @files
-    end
-
-    def watched_file_by_path(path)
-      index = @pointers[path]
-      return nil unless index
-      @files[index]
-    end
-
-    private
-
-    def last_modified_asc
-      @files.sort! do |left, right|
-        left.modified_at <=> right.modified_at
-      end
-      refresh_pointers
-    end
+    # @return [Enumerable<String>] managed path keys (snapshot)
+    alias keys paths
 
-    def last_modified_desc
-      @files.sort! do |left, right|
-        right.modified_at <=> left.modified_at
-      end
-      refresh_pointers
-    end
-
-    def path_asc
-      @files.sort! do |left, right|
-        left.path <=> right.path
-      end
-      refresh_pointers
-    end
+    # @return [Enumerable<WatchedFile>] managed files (snapshot)
+    alias values files
 
-    def path_desc
-      @files.sort! do |left, right|
-        right.path <=> left.path
-      end
-      refresh_pointers
-    end
-
-    def refresh_pointers
-      @files.each_with_index do |watched_file, index|
-        @pointers[watched_file.path] = index
-      end
-    end
   end
 end