logstash-input-file 4.1.18 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc2b7f39bcc8e3dfb894ea72af1b07e78fff73bf97ad12614a9ec440fed0c2d1
-  data.tar.gz: 6715064a8a25a13538cc07216f5c479f79dd2d3ba73a267377cd357c6e27ea8d
+  metadata.gz: 6cdc2495fe9eb7a2c84aaaac6ea922a6034a9ad1bd8223f021ed9bd77fedae74
+  data.tar.gz: 4e6ba1c9fc044ff6cd93cbc92c16d114415357877ea75ad2fce375514517f0d7
 SHA512:
-  metadata.gz: fbc9d7cd0f7b50be21721bfe0cb7c26a1e87a1f5697f4a01f14dcdc14a8449c9a9d14ddec58f3ec1b8bd611e624855b538155af8f2924604fb9f1cf120de4e22
-  data.tar.gz: 44b4bdb7b356a6494cb1cde6bf48395d61222234d2ec6cfb134d551a20ed1ab08043b23266b5bc75854b85577d22a1fd36d5b40c17213eb10df68f2903086146
+  metadata.gz: a790d7f2a79551d8e8a96ca8cee597c1baba4dd9eabff1f9cc806432072a58826c6066922431f020105d7d86a53b22a2a0136e3c132d1b4d532f844a4a815f2e
+  data.tar.gz: 0d4528eab5b7c56b9b6b62b7f0e7124c8bd47745ac46fb915210a7647021fd3fd0f97f4ef031d14fdd6ec03b28b697dd255c9ec9676d888cbf1107d987609c92

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.2.0
+  - Fix: watched files performance with huge filesets [#268](https://github.com/logstash-plugins/logstash-input-file/pull/268) 
+  - Updated logging to include full traces in debug (and trace) levels
+
 ## 4.1.18
   - Fix: release watched files on completion (in read-mode) [#271](https://github.com/logstash-plugins/logstash-input-file/pull/271)
 

data/lib/filewatch/discoverer.rb

@@ -9,6 +9,8 @@ module FileWatch
     # associated with a sincedb entry if one can be found
     include LogStash::Util::Loggable
 
+    attr_reader :watched_files_collection
+
     def initialize(watched_files_collection, sincedb_collection, settings)
       @watching = Concurrent::Array.new
       @exclude = Concurrent::Array.new
@@ -37,8 +39,7 @@ module FileWatch
       @exclude.each do |pattern|
         if watched_file.pathname.basename.fnmatch?(pattern)
           if new_discovery
-            logger.trace("Discoverer can_exclude?: #{watched_file.path}: skipping " +
-              "because it matches exclude #{pattern}")
+            logger.trace("skipping file because it matches exclude", :path => watched_file.path, :pattern => pattern)
           end
           watched_file.unwatch
           return true
@@ -56,13 +57,13 @@ module FileWatch
     end
 
     def discover_any_files(path, ongoing)
-      fileset = Dir.glob(path).select{|f| File.file?(f)}
-      logger.trace("discover_files",  "count" => fileset.size)
+      fileset = Dir.glob(path).select { |f| File.file?(f) }
+      logger.trace("discover_files", :count => fileset.size)
       fileset.each do |file|
-        pathname = Pathname.new(file)
         new_discovery = false
-        watched_file = @watched_files_collection.watched_file_by_path(file)
+        watched_file = @watched_files_collection.get(file)
         if watched_file.nil?
+          pathname = Pathname.new(file)
           begin
             path_stat = PathStatClass.new(pathname)
           rescue Errno::ENOENT
@@ -74,7 +75,7 @@ module FileWatch
         # if it already unwatched or its excluded then we can skip
         next if watched_file.unwatched? || can_exclude?(watched_file, new_discovery)
 
-        logger.trace("discover_files handling:", "new discovery"=> new_discovery, "watched_file details" => watched_file.details)
+        logger.trace? && logger.trace("handling:", :new_discovery => new_discovery, :watched_file => watched_file.details)
 
         if new_discovery
           watched_file.initial_completed if ongoing
@@ -86,7 +87,7 @@ module FileWatch
           #   associated with a different watched_file
           if @sincedb_collection.associate(watched_file)
             if watched_file.file_ignorable?
-              logger.trace("Discoverer discover_files: #{file}: skipping because it was last modified more than #{@settings.ignore_older} seconds ago")
+              logger.trace("skipping file because it was last modified more than #{@settings.ignore_older} seconds ago", :path => file)
               # on discovery ignorable watched_files are put into the ignored state and that
               # updates the size from the internal stat
               # so the existing contents are not read.

data/lib/filewatch/observing_base.rb

@@ -62,8 +62,7 @@ module FileWatch
       @sincedb_collection = SincedbCollection.new(@settings)
       @sincedb_collection.open
       discoverer = Discoverer.new(watched_files_collection, @sincedb_collection, @settings)
-      @watch = Watch.new(discoverer, watched_files_collection, @settings)
-      @watch.add_processor build_specific_processor(@settings)
+      @watch = Watch.new(discoverer, build_specific_processor(@settings), @settings)
     end
 
     def watch_this(path)

data/lib/filewatch/processor.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require 'concurrent/atomic/atomic_reference'
+
+module FileWatch
+  class Processor
+    include LogStash::Util::Loggable
+
+    attr_reader :watch
+
+    def initialize(settings)
+      @settings = settings
+      @deletable_paths = Concurrent::AtomicReference.new []
+    end
+
+    def add_watch(watch)
+      @watch = watch
+      self
+    end
+
+    def clear_deletable_paths
+      @deletable_paths.get_and_set []
+    end
+
+    def add_deletable_path(path)
+      @deletable_paths.get << path
+    end
+
+    def restat(watched_file)
+      changed = watched_file.restat!
+      if changed
+        # the collection (when sorted by modified_at) needs to re-sort every time watched-file is modified,
+        # we can perform these update operation while processing files (stat interval) instead of having to
+        # re-sort the whole collection every time an entry is accessed
+        @watch.watched_files_collection.update(watched_file)
+      end
+    end
+
+    private
+
+    def error_details(error, watched_file)
+      details = { :path => watched_file.path,
+                  :exception => error.class,
+                  :message => error.message,
+                  :backtrace => error.backtrace }
+      if logger.debug?
+        details[:file] = watched_file
+      else
+        details[:backtrace] = details[:backtrace].take(8) if details[:backtrace]
+      end
+      details
+    end
+
+  end
+end

data/lib/filewatch/read_mode/handlers/base.rb

@@ -34,19 +34,21 @@ module FileWatch module ReadMode module Handlers
 
     def open_file(watched_file)
       return true if watched_file.file_open?
-      logger.trace("opening #{watched_file.path}")
+      logger.trace? && logger.trace("opening", :path => watched_file.path)
       begin
         watched_file.open
-      rescue
+      rescue => e
         # don't emit this message too often. if a file that we can't
         # read is changing a lot, we'll try to open it more often, and spam the logs.
         now = Time.now.to_i
         logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
         if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
-          logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
+          backtrace = e.backtrace
+          backtrace = backtrace.take(3) if backtrace && !logger.debug?
+          logger.warn("failed to open", :path => watched_file.path, :exception => e.class, :message => e.message, :backtrace => backtrace)
           watched_file.last_open_warning_at = now
         else
-          logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
+          logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
         end
         watched_file.watch # set it back to watch so we can try it again
       end
@@ -75,7 +77,7 @@ module FileWatch module ReadMode module Handlers
           watched_file.update_bytes_read(sincedb_value.position)
         else
           sincedb_value.set_watched_file(watched_file)
-          logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details)
+          logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
           watched_file.rotate_from(existing_watched_file)
         end
 
@@ -92,7 +94,7 @@ module FileWatch module ReadMode module Handlers
     def add_new_value_sincedb_collection(watched_file)
       sincedb_value = SincedbValue.new(0)
       sincedb_value.set_watched_file(watched_file)
-      logger.trace("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position)
+      logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
       sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
     end
   end

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -31,7 +31,7 @@ module FileWatch module ReadMode module Handlers
     end
 
     def controlled_read(watched_file, loop_control)
-      logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename)
+      logger.trace? && logger.trace("reading...", :filename => watched_file.filename, :iterations => loop_control.count, :amount => loop_control.size)
       loop_control.count.times do
         break if quit?
         begin
@@ -43,22 +43,35 @@ module FileWatch module ReadMode module Handlers
             delta = line.bytesize + @settings.delimiter_byte_size
             sincedb_collection.increment(watched_file.sincedb_key, delta)
           end
-        rescue EOFError
-          logger.error("controlled_read: eof error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+        rescue EOFError => e
+          log_error("controlled_read: eof error reading file", watched_file, e)
           loop_control.flag_read_error
           break
-        rescue Errno::EWOULDBLOCK, Errno::EINTR
-          logger.error("controlled_read: block or interrupt error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+        rescue Errno::EWOULDBLOCK, Errno::EINTR => e
+          log_error("controlled_read: block or interrupt error reading file", watched_file, e)
           watched_file.listener.error
           loop_control.flag_read_error
           break
         rescue => e
-          logger.error("controlled_read: general error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+          log_error("controlled_read: general error reading file", watched_file, e)
           watched_file.listener.error
           loop_control.flag_read_error
           break
         end
       end
     end
+
+    def log_error(msg, watched_file, error)
+      details = { :path => watched_file.path,
+                  :exception => error.class,
+                  :message => error.message,
+                  :backtrace => error.backtrace }
+      if logger.debug?
+        details[:file] = watched_file
+      else
+        details[:backtrace] = details[:backtrace].take(8) if details[:backtrace]
+      end
+      logger.error(msg, details)
+    end
   end
 end end end

data/lib/filewatch/read_mode/handlers/read_zip_file.rb

@@ -1,13 +1,15 @@
 # encoding: utf-8
 require 'java'
-java_import java.io.InputStream
-java_import java.io.InputStreamReader
-java_import java.io.FileInputStream
-java_import java.io.BufferedReader
-java_import java.util.zip.GZIPInputStream
-java_import java.util.zip.ZipException
 
 module FileWatch module ReadMode module Handlers
+
+  java_import java.io.InputStream
+  java_import java.io.InputStreamReader
+  java_import java.io.FileInputStream
+  java_import java.io.BufferedReader
+  java_import java.util.zip.GZIPInputStream
+  java_import java.util.zip.ZipException
+
   class ReadZipFile < Base
     def handle_specifically(watched_file)
       add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)

data/lib/filewatch/read_mode/processor.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
-require "logstash/util/loggable"
-
+require 'filewatch/processor'
 require_relative "handlers/base"
 require_relative "handlers/read_file"
 require_relative "handlers/read_zip_file"
@@ -9,20 +8,7 @@ module FileWatch module ReadMode
   # Must handle
   #   :read_file
   #   :read_zip_file
-  class Processor
-    include LogStash::Util::Loggable
-
-    attr_reader :watch, :deletable_filepaths
-
-    def initialize(settings)
-      @settings = settings
-      @deletable_filepaths = []
-    end
-
-    def add_watch(watch)
-      @watch = watch
-      self
-    end
+  class Processor < FileWatch::Processor
 
     def initialize_handlers(sincedb_collection, observer)
       # we deviate from the tail mode handler initialization here
@@ -48,24 +34,23 @@ module FileWatch module ReadMode
     private
 
     def process_watched(watched_files)
-      logger.trace("Watched processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the watched state.
       # for a slice of them:
       #   move to the active state
       #   should never have been active before
       # how much of the max active window is available
-      to_take = @settings.max_active - watched_files.count{|wf| wf.active?}
+      to_take = @settings.max_active - watched_files.count { |wf| wf.active? }
       if to_take > 0
-        watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file|
-          path = watched_file.path
+        watched_files.select(&:watched?).take(to_take).each do |watched_file|
           begin
-            watched_file.restat
+            restat(watched_file)
             watched_file.activate
           rescue Errno::ENOENT
-            common_deleted_reaction(watched_file, "Watched")
+            common_deleted_reaction(watched_file, __method__)
             next
           rescue => e
-            common_error_reaction(path, e, "Watched")
+            common_error_reaction(watched_file, e, __method__)
             next
           end
           break if watch.quit?
@@ -74,7 +59,7 @@ module FileWatch module ReadMode
         now = Time.now.to_i
         if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL
           waiting = watched_files.size - @settings.max_active
-          logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}")
+          logger.warn("#{@settings.max_warn_msg}, files yet to open: #{waiting}")
           watch.lastwarn_max_files = now
         end
       end
@@ -83,17 +68,18 @@ module FileWatch module ReadMode
     ## TODO add process_rotation_in_progress
 
     def process_active(watched_files)
-      logger.trace("Active processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the active state.
-      watched_files.select {|wf| wf.active? }.each do |watched_file|
-        path = watched_file.path
+      watched_files.each do |watched_file|
+        next unless watched_file.active?
+
         begin
-          watched_file.restat
+          restat(watched_file)
         rescue Errno::ENOENT
-          common_deleted_reaction(watched_file, "Active")
+          common_deleted_reaction(watched_file, __method__)
           next
         rescue => e
-          common_error_reaction(path, e, "Active")
+          common_error_reaction(watched_file, e, __method__)
           next
         end
         break if watch.quit?
@@ -114,19 +100,19 @@ module FileWatch module ReadMode
     def common_detach_when_allread(watched_file)
       watched_file.unwatch
       watched_file.listener.reading_completed
-      deletable_filepaths << watched_file.path
-      logger.trace("Whole file read: #{watched_file.path}, removing from collection")
+      add_deletable_path watched_file.path
+      logger.trace? && logger.trace("whole file read, removing from collection", :path => watched_file.path)
     end
 
     def common_deleted_reaction(watched_file, action)
       # file has gone away or we can't read it anymore.
       watched_file.unwatch
-      deletable_filepaths << watched_file.path
-      logger.trace("#{action} - stat failed: #{watched_file.path}, removing from collection")
+      add_deletable_path watched_file.path
+      logger.trace? && logger.trace("#{action} - stat failed, removing from collection", :path => watched_file.path)
     end
 
-    def common_error_reaction(path, error, action)
-      logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})")
+    def common_error_reaction(watched_file, error, action)
+      logger.error("#{action} - other error", error_details(error, watched_file))
     end
   end
 end end

data/lib/filewatch/sincedb_collection.rb

@@ -56,12 +56,12 @@ module FileWatch
         logger.trace("open: count of keys read: #{@sincedb.keys.size}")
       rescue => e
         #No existing sincedb to load
-        logger.trace("open: error: #{path}: #{e.inspect}")
+        logger.trace("open: error:", :path => path, :exception => e.class, :message => e.message)
       end
     end
 
     def associate(watched_file)
-      logger.trace("associate: finding", "inode" => watched_file.sincedb_key.inode, "path" => watched_file.path)
+      logger.trace? && logger.trace("associate: finding", :path => watched_file.path, :inode => watched_file.sincedb_key.inode)
       sincedb_value = find(watched_file)
       if sincedb_value.nil?
         # sincedb has no record of this inode
@@ -71,7 +71,8 @@ module FileWatch
         logger.trace("associate: unmatched")
         return true
       end
-      logger.trace("associate: found sincedb record", "filename" => watched_file.filename, "sincedb key" => watched_file.sincedb_key,"sincedb_value" => sincedb_value)
+      logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename,
+                                    :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
       if sincedb_value.watched_file.nil?
         # not associated
         if sincedb_value.path_in_sincedb.nil?
@@ -106,7 +107,8 @@ module FileWatch
       #   after the original is deleted
       # are not yet in the delete phase, let this play out
       existing_watched_file = sincedb_value.watched_file
-      logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename", "this watched_file details" => watched_file.details, "other watched_file details" => existing_watched_file.details)
+      logger.trace? && logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename",
+                                    :this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details)
       watched_file.rotation_in_progress
       true
     end
@@ -149,8 +151,8 @@ module FileWatch
     end
 
     def watched_file_deleted(watched_file)
-      return unless member?(watched_file.sincedb_key)
-      get(watched_file.sincedb_key).unset_watched_file
+      value = @sincedb[watched_file.sincedb_key]
+      value.unset_watched_file if value
     end
 
     def store_last_read(key, pos)
@@ -195,7 +197,7 @@ module FileWatch
       watched_file.initial_completed
       if watched_file.all_read?
         watched_file.ignore
-        logger.trace("handle_association fully read, ignoring.....", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
+        logger.trace? && logger.trace("handle_association fully read, ignoring.....", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
       end
     end
 
@@ -214,8 +216,8 @@ module FileWatch
         @write_method.call
         @serializer.expired_keys.each do |key|
           @sincedb[key].unset_watched_file
-          delete(key)
-          logger.trace("sincedb_write: cleaned", "key" => "'#{key}'")
+          delete(key) # delete
+          logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
         end
         @sincedb_last_write = time
         @write_requested = false

data/lib/filewatch/stat/generic.rb

@@ -3,24 +3,19 @@
 module FileWatch module Stat
   class Generic
 
-    attr_reader :identifier, :inode, :modified_at, :size, :inode_struct
+    attr_reader :inode, :modified_at, :size, :inode_struct
 
     def initialize(source)
-      @source = source
-      @identifier = nil
+      @source = source # Pathname
       restat
     end
 
-    def add_identifier(identifier) self; end
-
     def restat
-      @inner_stat = @source.stat
-      @inode = @inner_stat.ino.to_s
-      @modified_at = @inner_stat.mtime.to_f
-      @size = @inner_stat.size
-      @dev_major = @inner_stat.dev_major
-      @dev_minor = @inner_stat.dev_minor
-      @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor)
+      stat = @source.stat
+      @inode = stat.ino.to_s
+      @modified_at = stat.mtime.to_f
+      @size = stat.size
+      @inode_struct = InodeStruct.new(@inode, stat.dev_major, stat.dev_minor)
     end
 
     def windows?
@@ -28,7 +23,7 @@ module FileWatch module Stat
     end
 
     def inspect
-      "<Generic size='#{@size}', modified_at='#{@modified_at}', inode='#{@inode}', inode_struct='#{@inode_struct}'>"
+      "<#{self.class.name} size=#{@size}, modified_at=#{@modified_at}, inode='#{@inode}', inode_struct=#{@inode_struct}>"
     end
   end
 end end