logstash-input-file 4.1.17 → 4.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cdb425f23de3ef4ccd1a1358ce48789001145a9a4620405bac4dc6a75d2b7f69
-  data.tar.gz: b7c856e75936d99912a8a517b81e23be1603e4032a255e6340b2c4da7faa6639
+  metadata.gz: b9475678fd2320f61bf61a3e34884ea6512072934e96f2d16c08a4055ad5f8a9
+  data.tar.gz: ca48090438f9ba71d20cbaf24111fa479fabe2279b3704d092aa4df36c2680e9
 SHA512:
-  metadata.gz: d0641b7f747964983a6f2e3b8c68a19efc74bb085e6a635cc110cd8d1c43efe330a899a951933f228cf59015785cc00730bd196a84a63dd36241045d36614ebf
-  data.tar.gz: 834e9ad07ed15beaee47e0cd3d6d6f5e5494e41a6d5a2f4871f79df4d63837ef3c480f033eaac0d092794f4c0a36fe4d3466728d98cf2ad55ac564cd8dec0f4f
+  metadata.gz: 97b642eed6db061be8fc620ff66e4b89e5898fd23bad138407509db20294ec013bba3a4dfe10dfc4587bdea2c09f8f99cd468db38d46a5a82028be5c85730801
+  data.tar.gz: b1583d047f9c042133584f3a82673b661c1eb59f7586a7c39df807a5ec391785dad2230df8561e339be673ef3f799c65e1282eac369459fccbbc491227882121

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 4.2.3
+  - Refactor: improve debug logging (log catched exceptions) [#280](https://github.com/logstash-plugins/logstash-input-file/pull/280)
+
+## 4.2.2
+  - Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276)
+
+## 4.2.1
+  - Fix: skip sincedb eviction if read mode completion deletes file during flush [#273](https://github.com/logstash-plugins/logstash-input-file/pull/273)
+  
+## 4.2.0
+  - Fix: watched files performance with huge filesets [#268](https://github.com/logstash-plugins/logstash-input-file/pull/268) 
+  - Updated logging to include full traces in debug (and trace) levels
+
+## 4.1.18
+  - Fix: release watched files on completion (in read-mode) [#271](https://github.com/logstash-plugins/logstash-input-file/pull/271)
+
 ## 4.1.17
   - Added configuration setting `check_archive_validity` settings to enable
   gzipped files verification, issue

data/lib/filewatch/discoverer.rb

@@ -9,6 +9,8 @@ module FileWatch
     # associated with a sincedb entry if one can be found
     include LogStash::Util::Loggable
 
+    attr_reader :watched_files_collection
+
     def initialize(watched_files_collection, sincedb_collection, settings)
       @watching = Concurrent::Array.new
       @exclude = Concurrent::Array.new
@@ -37,8 +39,7 @@ module FileWatch
       @exclude.each do |pattern|
         if watched_file.pathname.basename.fnmatch?(pattern)
           if new_discovery
-            logger.trace("Discoverer can_exclude?: #{watched_file.path}: skipping " +
-              "because it matches exclude #{pattern}")
+            logger.trace("skipping file because it matches exclude", :path => watched_file.path, :pattern => pattern)
           end
           watched_file.unwatch
           return true
@@ -56,13 +57,13 @@ module FileWatch
     end
 
     def discover_any_files(path, ongoing)
-      fileset = Dir.glob(path).select{|f| File.file?(f)}
-      logger.trace("discover_files",  "count" => fileset.size)
+      fileset = Dir.glob(path).select { |f| File.file?(f) }
+      logger.trace("discover_files", :count => fileset.size)
       fileset.each do |file|
-        pathname = Pathname.new(file)
         new_discovery = false
-        watched_file = @watched_files_collection.watched_file_by_path(file)
+        watched_file = @watched_files_collection.get(file)
         if watched_file.nil?
+          pathname = Pathname.new(file)
           begin
             path_stat = PathStatClass.new(pathname)
           rescue Errno::ENOENT
@@ -74,7 +75,7 @@ module FileWatch
         # if it already unwatched or its excluded then we can skip
         next if watched_file.unwatched? || can_exclude?(watched_file, new_discovery)
 
-        logger.trace("discover_files handling:", "new discovery"=> new_discovery, "watched_file details" => watched_file.details)
+        logger.trace? && logger.trace("handling:", :new_discovery => new_discovery, :watched_file => watched_file.details)
 
         if new_discovery
           watched_file.initial_completed if ongoing
@@ -86,7 +87,7 @@ module FileWatch
           #   associated with a different watched_file
           if @sincedb_collection.associate(watched_file)
             if watched_file.file_ignorable?
-              logger.trace("Discoverer discover_files: #{file}: skipping because it was last modified more than #{@settings.ignore_older} seconds ago")
+              logger.trace("skipping file because it was last modified more than #{@settings.ignore_older} seconds ago", :path => file)
               # on discovery ignorable watched_files are put into the ignored state and that
               # updates the size from the internal stat
               # so the existing contents are not read.

data/lib/filewatch/observing_base.rb

@@ -62,8 +62,7 @@ module FileWatch
       @sincedb_collection = SincedbCollection.new(@settings)
       @sincedb_collection.open
       discoverer = Discoverer.new(watched_files_collection, @sincedb_collection, @settings)
-      @watch = Watch.new(discoverer, watched_files_collection, @settings)
-      @watch.add_processor build_specific_processor(@settings)
+      @watch = Watch.new(discoverer, build_specific_processor(@settings), @settings)
     end
 
     def watch_this(path)
@@ -84,15 +83,5 @@ module FileWatch
       # sincedb_write("shutting down")
     end
 
-    # close_file(path) is to be used by external code
-    # when it knows that it is completely done with a file.
-    # Other files or folders may still be being watched.
-    # Caution, once unwatched, a file can't be watched again
-    # unless a new instance of this class begins watching again.
-    # The sysadmin should rename, move or delete the file.
-    def close_file(path)
-      @watch.unwatch(path)
-      sincedb_write
-    end
   end
 end

data/lib/filewatch/processor.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require 'concurrent/atomic/atomic_reference'
+
+module FileWatch
+  class Processor
+    include LogStash::Util::Loggable
+
+    attr_reader :watch
+
+    def initialize(settings)
+      @settings = settings
+      @deletable_paths = Concurrent::AtomicReference.new []
+    end
+
+    def add_watch(watch)
+      @watch = watch
+      self
+    end
+
+    def clear_deletable_paths
+      @deletable_paths.get_and_set []
+    end
+
+    def add_deletable_path(path)
+      @deletable_paths.get << path
+    end
+
+    def restat(watched_file)
+      changed = watched_file.restat!
+      if changed
+        # the collection (when sorted by modified_at) needs to re-sort every time watched-file is modified,
+        # we can perform these update operation while processing files (stat interval) instead of having to
+        # re-sort the whole collection every time an entry is accessed
+        @watch.watched_files_collection.update(watched_file)
+      end
+    end
+
+    private
+
+    def error_details(error, watched_file)
+      details = { :path => watched_file.path,
+                  :exception => error.class,
+                  :message => error.message,
+                  :backtrace => error.backtrace }
+      if logger.debug?
+        details[:file] = watched_file
+      else
+        details[:backtrace] = details[:backtrace].take(8) if details[:backtrace]
+      end
+      details
+    end
+
+  end
+end

data/lib/filewatch/read_mode/handlers/base.rb

@@ -19,7 +19,7 @@ module FileWatch module ReadMode module Handlers
     end
 
     def handle(watched_file)
-      logger.trace("handling: #{watched_file.path}")
+      logger.trace? && logger.trace("handling:", :path => watched_file.path)
       unless watched_file.has_listener?
         watched_file.set_listener(@observer)
       end
@@ -34,19 +34,21 @@ module FileWatch module ReadMode module Handlers
 
     def open_file(watched_file)
       return true if watched_file.file_open?
-      logger.trace("opening #{watched_file.path}")
+      logger.trace? && logger.trace("opening", :path => watched_file.path)
       begin
         watched_file.open
-      rescue
+      rescue => e
         # don't emit this message too often. if a file that we can't
         # read is changing a lot, we'll try to open it more often, and spam the logs.
         now = Time.now.to_i
-        logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
+        logger.trace? && logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
         if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
-          logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
+          backtrace = e.backtrace
+          backtrace = backtrace.take(3) if backtrace && !logger.debug?
+          logger.warn("failed to open", :path => watched_file.path, :exception => e.class, :message => e.message, :backtrace => backtrace)
           watched_file.last_open_warning_at = now
         else
-          logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
+          logger.trace? && logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
         end
         watched_file.watch # set it back to watch so we can try it again
       end
@@ -65,8 +67,7 @@ module FileWatch module ReadMode module Handlers
       elsif sincedb_value.watched_file == watched_file
         update_existing_sincedb_collection_value(watched_file, sincedb_value)
       else
-        msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
-        logger.trace(msg)
+        logger.trace? && logger.trace("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
         existing_watched_file = sincedb_value.watched_file
         if existing_watched_file.nil?
           sincedb_value.set_watched_file(watched_file)
@@ -75,7 +76,7 @@ module FileWatch module ReadMode module Handlers
           watched_file.update_bytes_read(sincedb_value.position)
         else
           sincedb_value.set_watched_file(watched_file)
-          logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details)
+          logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
           watched_file.rotate_from(existing_watched_file)
         end
 
@@ -84,7 +85,7 @@ module FileWatch module ReadMode module Handlers
     end
 
     def update_existing_sincedb_collection_value(watched_file, sincedb_value)
-      logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
+      logger.trace? && logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
       # sincedb_value is the source of truth
       watched_file.update_bytes_read(sincedb_value.position)
     end
@@ -92,7 +93,7 @@ module FileWatch module ReadMode module Handlers
     def add_new_value_sincedb_collection(watched_file)
       sincedb_value = SincedbValue.new(0)
       sincedb_value.set_watched_file(watched_file)
-      logger.trace("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position)
+      logger.trace? && logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
       sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
     end
   end

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -19,16 +19,21 @@ module FileWatch module ReadMode module Handlers
           watched_file.listener.eof
           watched_file.file_close
           key = watched_file.sincedb_key
-          sincedb_collection.reading_completed(key)
-          sincedb_collection.clear_watched_file(key)
+          if sincedb_collection.get(key)
+            sincedb_collection.reading_completed(key)
+            sincedb_collection.clear_watched_file(key)
+          end
           watched_file.listener.deleted
+          # NOTE: on top of un-watching we should also remove from the watched files collection
+          # if the file is getting deleted (on completion), that part currently resides in
+          # DeleteCompletedFileHandler - triggered above using `watched_file.listener.deleted`
           watched_file.unwatch
         end
       end
     end
 
     def controlled_read(watched_file, loop_control)
-      logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename)
+      logger.trace? && logger.trace("reading...", :filename => watched_file.filename, :iterations => loop_control.count, :amount => loop_control.size)
       loop_control.count.times do
         break if quit?
         begin
@@ -40,22 +45,35 @@ module FileWatch module ReadMode module Handlers
             delta = line.bytesize + @settings.delimiter_byte_size
             sincedb_collection.increment(watched_file.sincedb_key, delta)
           end
-        rescue EOFError
-          logger.error("controlled_read: eof error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+        rescue EOFError => e
+          log_error("controlled_read: eof error reading file", watched_file, e)
           loop_control.flag_read_error
           break
-        rescue Errno::EWOULDBLOCK, Errno::EINTR
-          logger.error("controlled_read: block or interrupt error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+        rescue Errno::EWOULDBLOCK, Errno::EINTR => e
+          log_error("controlled_read: block or interrupt error reading file", watched_file, e)
           watched_file.listener.error
           loop_control.flag_read_error
           break
         rescue => e
-          logger.error("controlled_read: general error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+          log_error("controlled_read: general error reading file", watched_file, e)
           watched_file.listener.error
           loop_control.flag_read_error
           break
         end
       end
     end
+
+    def log_error(msg, watched_file, error)
+      details = { :path => watched_file.path,
+                  :exception => error.class,
+                  :message => error.message,
+                  :backtrace => error.backtrace }
+      if logger.debug?
+        details[:file] = watched_file
+      else
+        details[:backtrace] = details[:backtrace].take(8) if details[:backtrace]
+      end
+      logger.error(msg, details)
+    end
   end
 end end end

data/lib/filewatch/read_mode/handlers/read_zip_file.rb

@@ -1,13 +1,15 @@
 # encoding: utf-8
 require 'java'
-java_import java.io.InputStream
-java_import java.io.InputStreamReader
-java_import java.io.FileInputStream
-java_import java.io.BufferedReader
-java_import java.util.zip.GZIPInputStream
-java_import java.util.zip.ZipException
 
 module FileWatch module ReadMode module Handlers
+
+  java_import java.io.InputStream
+  java_import java.io.InputStreamReader
+  java_import java.io.FileInputStream
+  java_import java.io.BufferedReader
+  java_import java.util.zip.GZIPInputStream
+  java_import java.util.zip.ZipException
+
   class ReadZipFile < Base
     def handle_specifically(watched_file)
       add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)

data/lib/filewatch/read_mode/processor.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
-require "logstash/util/loggable"
-
+require 'filewatch/processor'
 require_relative "handlers/base"
 require_relative "handlers/read_file"
 require_relative "handlers/read_zip_file"
@@ -9,20 +8,7 @@ module FileWatch module ReadMode
   # Must handle
   #   :read_file
   #   :read_zip_file
-  class Processor
-    include LogStash::Util::Loggable
-
-    attr_reader :watch, :deletable_filepaths
-
-    def initialize(settings)
-      @settings = settings
-      @deletable_filepaths = []
-    end
-
-    def add_watch(watch)
-      @watch = watch
-      self
-    end
+  class Processor < FileWatch::Processor
 
     def initialize_handlers(sincedb_collection, observer)
       # we deviate from the tail mode handler initialization here
@@ -48,24 +34,23 @@ module FileWatch module ReadMode
     private
 
     def process_watched(watched_files)
-      logger.trace("Watched processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the watched state.
       # for a slice of them:
       #   move to the active state
       #   should never have been active before
       # how much of the max active window is available
-      to_take = @settings.max_active - watched_files.count{|wf| wf.active?}
+      to_take = @settings.max_active - watched_files.count { |wf| wf.active? }
       if to_take > 0
-        watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file|
-          path = watched_file.path
+        watched_files.select(&:watched?).take(to_take).each do |watched_file|
           begin
-            watched_file.restat
+            restat(watched_file)
             watched_file.activate
           rescue Errno::ENOENT
-            common_deleted_reaction(watched_file, "Watched")
+            common_deleted_reaction(watched_file, __method__)
             next
           rescue => e
-            common_error_reaction(path, e, "Watched")
+            common_error_reaction(watched_file, e, __method__)
             next
           end
           break if watch.quit?
@@ -74,7 +59,7 @@ module FileWatch module ReadMode
         now = Time.now.to_i
         if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL
           waiting = watched_files.size - @settings.max_active
-          logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}")
+          logger.warn("#{@settings.max_warn_msg}, files yet to open: #{waiting}")
           watch.lastwarn_max_files = now
         end
       end
@@ -83,17 +68,18 @@ module FileWatch module ReadMode
     ## TODO add process_rotation_in_progress
 
     def process_active(watched_files)
-      logger.trace("Active processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the active state.
-      watched_files.select {|wf| wf.active? }.each do |watched_file|
-        path = watched_file.path
+      watched_files.each do |watched_file|
+        next unless watched_file.active?
+
         begin
-          watched_file.restat
+          restat(watched_file)
         rescue Errno::ENOENT
-          common_deleted_reaction(watched_file, "Active")
+          common_deleted_reaction(watched_file, __method__)
           next
         rescue => e
-          common_error_reaction(path, e, "Active")
+          common_error_reaction(watched_file, e, __method__)
           next
         end
         break if watch.quit?
@@ -114,19 +100,19 @@ module FileWatch module ReadMode
     def common_detach_when_allread(watched_file)
       watched_file.unwatch
       watched_file.listener.reading_completed
-      deletable_filepaths << watched_file.path
-      logger.trace("Whole file read: #{watched_file.path}, removing from collection")
+      add_deletable_path watched_file.path
+      logger.trace? && logger.trace("whole file read, removing from collection", :path => watched_file.path)
     end
 
     def common_deleted_reaction(watched_file, action)
       # file has gone away or we can't read it anymore.
       watched_file.unwatch
-      deletable_filepaths << watched_file.path
-      logger.trace("#{action} - stat failed: #{watched_file.path}, removing from collection")
+      add_deletable_path watched_file.path
+      logger.trace? && logger.trace("#{action} - stat failed, removing from collection", :path => watched_file.path)
     end
 
-    def common_error_reaction(path, error, action)
-      logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})")
+    def common_error_reaction(watched_file, error, action)
+      logger.error("#{action} - other error", error_details(error, watched_file))
     end
   end
 end end

data/lib/filewatch/settings.rb

@@ -6,7 +6,7 @@ module FileWatch
     attr_reader :max_active, :max_warn_msg, :lastwarn_max_files
     attr_reader :sincedb_write_interval, :stat_interval, :discover_interval
     attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size
-    attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration
+    attr_reader :sincedb_path, :sincedb_expiry_duration
     attr_reader :file_sort_by, :file_sort_direction
     attr_reader :exit_after_read
     attr_reader :check_archive_validity
@@ -41,7 +41,6 @@ module FileWatch
       @file_chunk_size = @opts[:file_chunk_size]
       @close_older = @opts[:close_older]
       @ignore_older = @opts[:ignore_older]
-      @sincedb_write_interval = @opts[:sincedb_write_interval]
       @stat_interval = @opts[:stat_interval]
       @discover_interval = @opts[:discover_interval]
       @exclude = Array(@opts[:exclude])