logstash-input-file 4.1.18 → 4.2.4

data/lib/filewatch/settings.rb

@@ -6,7 +6,7 @@ module FileWatch
     attr_reader :max_active, :max_warn_msg, :lastwarn_max_files
     attr_reader :sincedb_write_interval, :stat_interval, :discover_interval
     attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size
-    attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration
+    attr_reader :sincedb_path, :sincedb_expiry_duration
     attr_reader :file_sort_by, :file_sort_direction
     attr_reader :exit_after_read
     attr_reader :check_archive_validity
@@ -41,7 +41,6 @@ module FileWatch
       @file_chunk_size = @opts[:file_chunk_size]
       @close_older = @opts[:close_older]
       @ignore_older = @opts[:ignore_older]
-      @sincedb_write_interval = @opts[:sincedb_write_interval]
       @stat_interval = @opts[:stat_interval]
       @discover_interval = @opts[:discover_interval]
       @exclude = Array(@opts[:exclude])

data/lib/filewatch/sincedb_collection.rb

@@ -47,55 +47,53 @@ module FileWatch
       @time_sdb_opened = Time.now.to_f
       begin
         path.open do |file|
-          logger.trace("open: reading from #{path}")
+          logger.debug("open: reading from #{path}")
           @serializer.deserialize(file) do |key, value|
-            logger.trace("open: importing ... '#{key}' => '#{value}'")
+            logger.trace? && logger.trace("open: importing #{key.inspect} => #{value.inspect}")
             set_key_value(key, value)
           end
         end
         logger.trace("open: count of keys read: #{@sincedb.keys.size}")
       rescue => e
         #No existing sincedb to load
-        logger.trace("open: error: #{path}: #{e.inspect}")
+        logger.debug("open: error opening #{path}", :exception => e.class, :message => e.message)
       end
     end
 
     def associate(watched_file)
-      logger.trace("associate: finding", "inode" => watched_file.sincedb_key.inode, "path" => watched_file.path)
+      logger.trace? && logger.trace("associate: finding", :path => watched_file.path, :inode => watched_file.sincedb_key.inode)
       sincedb_value = find(watched_file)
       if sincedb_value.nil?
         # sincedb has no record of this inode
         # and due to the window handling of many files
         # this file may not be opened in this session.
         # a new value will be added when the file is opened
-        logger.trace("associate: unmatched")
+        logger.trace("associate: unmatched", :filename => watched_file.filename)
         return true
       end
-      logger.trace("associate: found sincedb record", "filename" => watched_file.filename, "sincedb key" => watched_file.sincedb_key,"sincedb_value" => sincedb_value)
-      if sincedb_value.watched_file.nil?
-        # not associated
+      logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename,
+                                    :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
+      if sincedb_value.watched_file.nil? # not associated
         if sincedb_value.path_in_sincedb.nil?
           handle_association(sincedb_value, watched_file)
-          logger.trace("associate: inode matched but no path in sincedb")
+          logger.trace? && logger.trace("associate: inode matched but no path in sincedb", :filename => watched_file.filename)
           return true
         end
         if sincedb_value.path_in_sincedb == watched_file.path
-          # the path on disk is the same as discovered path
-          # and the inode is the same.
+          # the path on disk is the same as discovered path and the inode is the same.
           handle_association(sincedb_value, watched_file)
-          logger.trace("associate: inode and path matched")
+          logger.trace? && logger.trace("associate: inode and path matched", :filename => watched_file.filename)
           return true
         end
-        # the path on disk is different from discovered unassociated path
-        # but they have the same key (inode)
+        # the path on disk is different from discovered unassociated path but they have the same key (inode)
         # treat as a new file, a new value will be added when the file is opened
         sincedb_value.clear_watched_file
         delete(watched_file.sincedb_key)
-        logger.trace("associate: matched but allocated to another")
+        logger.trace? && logger.trace("associate: matched but allocated to another", :filename => watched_file.filename)
         return true
       end
       if sincedb_value.watched_file.equal?(watched_file) # pointer equals
-        logger.trace("associate: already associated")
+        logger.trace? && logger.trace("associate: already associated", :filename => watched_file.filename)
         return true
       end
       # sincedb_value.watched_file is not this discovered watched_file but they have the same key (inode)
@@ -106,7 +104,8 @@ module FileWatch
       #   after the original is deleted
       # are not yet in the delete phase, let this play out
       existing_watched_file = sincedb_value.watched_file
-      logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename", "this watched_file details" => watched_file.details, "other watched_file details" => existing_watched_file.details)
+      logger.trace? && logger.trace("associate: found sincedb_value has a watched_file - this is a rename",
+                                    :this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details)
       watched_file.rotation_in_progress
       true
     end
@@ -149,8 +148,8 @@ module FileWatch
     end
 
     def watched_file_deleted(watched_file)
-      return unless member?(watched_file.sincedb_key)
-      get(watched_file.sincedb_key).unset_watched_file
+      value = @sincedb[watched_file.sincedb_key]
+      value.unset_watched_file if value
     end
 
     def store_last_read(key, pos)
@@ -178,63 +177,63 @@ module FileWatch
       get(key).watched_file.nil?
     end
 
-    private
-
     def flush_at_interval
-      now = Time.now.to_i
-      delta = now - @sincedb_last_write
+      now = Time.now
+      delta = now.to_i - @sincedb_last_write
       if delta >= @settings.sincedb_write_interval
         logger.debug("writing sincedb (delta since last write = #{delta})")
         sincedb_write(now)
       end
     end
 
+    private
+
     def handle_association(sincedb_value, watched_file)
       watched_file.update_bytes_read(sincedb_value.position)
       sincedb_value.set_watched_file(watched_file)
       watched_file.initial_completed
       if watched_file.all_read?
         watched_file.ignore
-        logger.trace("handle_association fully read, ignoring.....", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
+        logger.trace? && logger.trace("handle_association fully read, ignoring", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
       end
     end
 
     def set_key_value(key, value)
       if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration)
-        logger.trace("open: setting #{key.inspect} to #{value.inspect}")
         set(key, value)
       else
-        logger.trace("open: record has expired, skipping: #{key.inspect} #{value.inspect}")
+        logger.debug("set_key_value: record has expired, skipping: #{key.inspect} => #{value.inspect}")
       end
     end
 
-    def sincedb_write(time = Time.now.to_i)
-      logger.trace("sincedb_write: to: #{path}")
+    def sincedb_write(time = Time.now)
+      logger.trace? && logger.trace("sincedb_write: #{path} (time = #{time})")
       begin
-        @write_method.call
-        @serializer.expired_keys.each do |key|
+        expired_keys = @write_method.call(time)
+        expired_keys.each do |key|
           @sincedb[key].unset_watched_file
           delete(key)
-          logger.trace("sincedb_write: cleaned", "key" => "'#{key}'")
+          logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
         end
-        @sincedb_last_write = time
+        @sincedb_last_write = time.to_i
         @write_requested = false
-      rescue Errno::EACCES
-        # no file handles free perhaps
-        # maybe it will work next time
-        logger.trace("sincedb_write: error: #{path}: #{$!}")
+      rescue Errno::EACCES => e
+        # no file handles free perhaps - maybe it will work next time
+        logger.debug("sincedb_write: #{path} error:", :exception => e.class, :message => e.message)
       end
     end
 
-    def atomic_write
+    # @return expired keys
+    def atomic_write(time)
       FileHelper.write_atomically(@full_path) do |io|
-        @serializer.serialize(@sincedb, io)
+        @serializer.serialize(@sincedb, io, time.to_f)
       end
     end
 
-    def non_atomic_write
-      IO.open(IO.sysopen(@full_path, "w+")) do |io|
-        @serializer.serialize(@sincedb, io)
+    # @return expired keys
+    def non_atomic_write(time)
+      File.open(@full_path, "w+") do |io|
+        @serializer.serialize(@sincedb, io, time.to_f)
       end
     end
   end

data/lib/filewatch/sincedb_record_serializer.rb

@@ -3,30 +3,25 @@
 module FileWatch
   class SincedbRecordSerializer
 
-    attr_reader :expired_keys
-
     def self.days_to_seconds(days)
       (24 * 3600) * days.to_f
     end
 
     def initialize(sincedb_value_expiry)
       @sincedb_value_expiry = sincedb_value_expiry
-      @expired_keys = []
-    end
-
-    def update_sincedb_value_expiry_from_days(days)
-      @sincedb_value_expiry = SincedbRecordSerializer.days_to_seconds(days)
     end
 
+    # @return Array expired keys (ones that were not written to the file)
     def serialize(db, io, as_of = Time.now.to_f)
-      @expired_keys.clear
+      expired_keys = []
       db.each do |key, value|
         if as_of > value.last_changed_at_expires(@sincedb_value_expiry)
-          @expired_keys << key
+          expired_keys << key
           next
         end
         io.write(serialize_record(key, value))
       end
+      expired_keys
     end
 
     def deserialize(io)
@@ -36,8 +31,7 @@ module FileWatch
     end
 
     def serialize_record(k, v)
-      # effectively InodeStruct#to_s SincedbValue#to_s
-      "#{k} #{v}\n"
+      "#{k} #{v}\n" # effectively InodeStruct#to_s SincedbValue#to_s
     end
 
     def deserialize_record(record)

data/lib/filewatch/stat/generic.rb

@@ -3,24 +3,19 @@
 module FileWatch module Stat
   class Generic
 
-    attr_reader :identifier, :inode, :modified_at, :size, :inode_struct
+    attr_reader :inode, :modified_at, :size, :inode_struct
 
     def initialize(source)
-      @source = source
-      @identifier = nil
+      @source = source # Pathname
       restat
     end
 
-    def add_identifier(identifier) self; end
-
     def restat
-      @inner_stat = @source.stat
-      @inode = @inner_stat.ino.to_s
-      @modified_at = @inner_stat.mtime.to_f
-      @size = @inner_stat.size
-      @dev_major = @inner_stat.dev_major
-      @dev_minor = @inner_stat.dev_minor
-      @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor)
+      stat = @source.stat
+      @inode = stat.ino.to_s
+      @modified_at = stat.mtime.to_f
+      @size = stat.size
+      @inode_struct = InodeStruct.new(@inode, stat.dev_major, stat.dev_minor)
     end
 
     def windows?
@@ -28,7 +23,7 @@ module FileWatch module Stat
     end
 
     def inspect
-      "<Generic size='#{@size}', modified_at='#{@modified_at}', inode='#{@inode}', inode_struct='#{@inode_struct}'>"
+      "<#{self.class.name} size=#{@size}, modified_at=#{@modified_at}, inode='#{@inode}', inode_struct=#{@inode_struct}>"
     end
   end
 end end

data/lib/filewatch/stat/windows_path.rb

@@ -3,22 +3,20 @@
 module FileWatch module Stat
   class WindowsPath
 
-    attr_reader :identifier, :inode, :modified_at, :size, :inode_struct
+    attr_reader :inode, :modified_at, :size, :inode_struct
 
     def initialize(source)
-      @source = source
+      @source = source # Pathname
       @inode = Winhelper.identifier_from_path(@source.to_path)
-      @dev_major = 0
-      @dev_minor = 0
       # in windows the dev hi and low are in the identifier
-      @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor)
+      @inode_struct = InodeStruct.new(@inode, 0, 0)
       restat
     end
 
     def restat
-      @inner_stat = @source.stat
-      @modified_at = @inner_stat.mtime.to_f
-      @size = @inner_stat.size
+      stat = @source.stat
+      @modified_at = stat.mtime.to_f
+      @size = stat.size
     end
 
     def windows?
@@ -26,7 +24,7 @@ module FileWatch module Stat
     end
 
     def inspect
-      "<WindowsPath size='#{@size}', modified_at='#{@modified_at}', inode='#{@inode}', inode_struct='#{@inode_struct}'>"
+      "<#{self.class.name} size=#{@size}, modified_at=#{@modified_at}, inode=#{@inode}, inode_struct=#{@inode_struct}>"
     end
   end
 end end

data/lib/filewatch/tail_mode/handlers/base.rb

@@ -18,7 +18,7 @@ module FileWatch module TailMode module Handlers
     end
 
     def handle(watched_file)
-      logger.trace("handling: #{watched_file.filename}")
+      logger.trace? && logger.trace("handling:", :path => watched_file.path)
       unless watched_file.has_listener?
         watched_file.set_listener(@observer)
       end
@@ -37,7 +37,7 @@ module FileWatch module TailMode module Handlers
 
     def controlled_read(watched_file, loop_control)
       changed = false
-      logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename)
+      logger.trace? && logger.trace(__method__.to_s, :iterations => loop_control.count, :amount => loop_control.size, :filename => watched_file.filename)
       # from a real config (has 102 file inputs)
       # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously
       # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF
@@ -48,7 +48,7 @@ module FileWatch module TailMode module Handlers
       loop_control.count.times do
         break if quit?
         begin
-          logger.debug("read_to_eof: get chunk")
+          logger.debug? && logger.debug("#{__method__} get chunk")
           result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult
           logger.trace(result.warning, result.additional) unless result.warning.empty?
           changed = true
@@ -57,40 +57,42 @@ module FileWatch module TailMode module Handlers
             # sincedb position is now independent from the watched_file bytes_read
             sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
           end
-        rescue EOFError
+        rescue EOFError => e
           # it only makes sense to signal EOF in "read" mode not "tail"
+          logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
           loop_control.flag_read_error
           break
-        rescue Errno::EWOULDBLOCK, Errno::EINTR
+        rescue Errno::EWOULDBLOCK, Errno::EINTR => e
+          logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
           watched_file.listener.error
           loop_control.flag_read_error
           break
         rescue => e
-          logger.error("read_to_eof: general error reading #{watched_file.path}", "error" => e.inspect, "backtrace" => e.backtrace.take(4))
+          logger.error("#{__method__} general error reading", exception_details(watched_file.path, e))
           watched_file.listener.error
           loop_control.flag_read_error
           break
         end
       end
-      logger.debug("read_to_eof: exit due to quit") if quit?
+      logger.debug("#{__method__} stopped loop due quit") if quit?
       sincedb_collection.request_disk_flush if changed
     end
 
     def open_file(watched_file)
       return true if watched_file.file_open?
-      logger.trace("opening #{watched_file.filename}")
+      logger.trace? && logger.trace("open_file", :filename => watched_file.filename)
       begin
         watched_file.open
-      rescue
+      rescue => e
         # don't emit this message too often. if a file that we can't
         # read is changing a lot, we'll try to open it more often, and spam the logs.
         now = Time.now.to_i
-        logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
+        logger.trace? && logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
         if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
-          logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
+          logger.warn("failed to open file", exception_details(watched_file.path, e))
           watched_file.last_open_warning_at = now
         else
-          logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
+          logger.debug("open_file suppressed warning `failed to open file`", exception_details(watched_file.path, e, false))
         end
         watched_file.watch # set it back to watch so we can try it again
       else
@@ -108,26 +110,22 @@ module FileWatch module TailMode module Handlers
         update_existing_sincedb_collection_value(watched_file, sincedb_value)
         watched_file.initial_completed
       else
-        msg = "add_or_update_sincedb_collection: found sincedb record"
-        logger.trace(msg,
-          "sincedb key" => watched_file.sincedb_key,
-          "sincedb value" => sincedb_value
-        )
+        logger.trace? && logger.trace("add_or_update_sincedb_collection: found sincedb record",
+                                      :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
         # detected a rotation, Discoverer can't handle this because this watched file is not a new discovery.
         # we must handle it here, by transferring state and have the sincedb value track this watched file
         # rotate_as_file and rotate_from will switch the sincedb key to the inode that the path is now pointing to
         # and pickup the sincedb_value from before.
-        msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
-        logger.trace(msg)
+        logger.debug("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
         existing_watched_file = sincedb_value.watched_file
         if existing_watched_file.nil?
           sincedb_value.set_watched_file(watched_file)
-          logger.trace("add_or_update_sincedb_collection: switching as new file")
+          logger.trace? && logger.trace("add_or_update_sincedb_collection: switching as new file")
           watched_file.rotate_as_file
           watched_file.update_bytes_read(sincedb_value.position)
         else
           sincedb_value.set_watched_file(watched_file)
-          logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details)
+          logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from:", :watched_file => watched_file.details)
           watched_file.rotate_from(existing_watched_file)
         end
       end
@@ -135,13 +133,15 @@ module FileWatch module TailMode module Handlers
     end
 
     def update_existing_sincedb_collection_value(watched_file, sincedb_value)
-      logger.trace("update_existing_sincedb_collection_value: #{watched_file.filename}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
+      logger.trace? && logger.trace("update_existing_sincedb_collection_value", :position => sincedb_value.position,
+                                    :filename => watched_file.filename, :last_stat_size => watched_file.last_stat_size)
       update_existing_specifically(watched_file, sincedb_value)
     end
 
     def add_new_value_sincedb_collection(watched_file)
       sincedb_value = get_new_value_specifically(watched_file)
-      logger.trace("add_new_value_sincedb_collection", "position" => sincedb_value.position, "watched_file details" => watched_file.details)
+      logger.trace? && logger.trace("add_new_value_sincedb_collection", :position => sincedb_value.position,
+                                    :watched_file => watched_file.details)
       sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
       sincedb_value
     end
@@ -153,5 +153,14 @@ module FileWatch module TailMode module Handlers
       watched_file.update_bytes_read(position)
       value
     end
+
+    private
+
+    def exception_details(path, e, trace = true)
+      details = { :path => path, :exception => e.class, :message => e.message }
+      details[:backtrace] = e.backtrace if trace && logger.debug?
+      details
+    end
+
   end
 end end end