logstash-input-file 4.1.16 → 4.2.2

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -19,16 +19,21 @@ module FileWatch module ReadMode module Handlers
           watched_file.listener.eof
           watched_file.file_close
           key = watched_file.sincedb_key
-          sincedb_collection.reading_completed(key)
-          sincedb_collection.clear_watched_file(key)
+          if sincedb_collection.get(key)
+            sincedb_collection.reading_completed(key)
+            sincedb_collection.clear_watched_file(key)
+          end
           watched_file.listener.deleted
+          # NOTE: on top of un-watching we should also remove from the watched files collection
+          # if the file is getting deleted (on completion), that part currently resides in
+          # DeleteCompletedFileHandler - triggered above using `watched_file.listener.deleted`
           watched_file.unwatch
         end
       end
     end
 
     def controlled_read(watched_file, loop_control)
-      logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename)
+      logger.trace? && logger.trace("reading...", :filename => watched_file.filename, :iterations => loop_control.count, :amount => loop_control.size)
       loop_control.count.times do
         break if quit?
         begin
@@ -40,22 +45,35 @@ module FileWatch module ReadMode module Handlers
             delta = line.bytesize + @settings.delimiter_byte_size
             sincedb_collection.increment(watched_file.sincedb_key, delta)
           end
-        rescue EOFError
-          logger.error("controlled_read: eof error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+        rescue EOFError => e
+          log_error("controlled_read: eof error reading file", watched_file, e)
           loop_control.flag_read_error
           break
-        rescue Errno::EWOULDBLOCK, Errno::EINTR
-          logger.error("controlled_read: block or interrupt error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+        rescue Errno::EWOULDBLOCK, Errno::EINTR => e
+          log_error("controlled_read: block or interrupt error reading file", watched_file, e)
           watched_file.listener.error
           loop_control.flag_read_error
           break
         rescue => e
-          logger.error("controlled_read: general error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8))
+          log_error("controlled_read: general error reading file", watched_file, e)
           watched_file.listener.error
           loop_control.flag_read_error
           break
         end
       end
     end
+
+    def log_error(msg, watched_file, error)
+      details = { :path => watched_file.path,
+                  :exception => error.class,
+                  :message => error.message,
+                  :backtrace => error.backtrace }
+      if logger.debug?
+        details[:file] = watched_file
+      else
+        details[:backtrace] = details[:backtrace].take(8) if details[:backtrace]
+      end
+      logger.error(msg, details)
+    end
   end
 end end end

data/lib/filewatch/read_mode/handlers/read_zip_file.rb

@@ -1,13 +1,15 @@
 # encoding: utf-8
 require 'java'
-java_import java.io.InputStream
-java_import java.io.InputStreamReader
-java_import java.io.FileInputStream
-java_import java.io.BufferedReader
-java_import java.util.zip.GZIPInputStream
-java_import java.util.zip.ZipException
 
 module FileWatch module ReadMode module Handlers
+
+  java_import java.io.InputStream
+  java_import java.io.InputStreamReader
+  java_import java.io.FileInputStream
+  java_import java.io.BufferedReader
+  java_import java.util.zip.GZIPInputStream
+  java_import java.util.zip.ZipException
+
   class ReadZipFile < Base
     def handle_specifically(watched_file)
       add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
@@ -18,34 +20,40 @@ module FileWatch module ReadMode module Handlers
       # fast forward through the lines until we reach unseen content?
       # meaning that we can quit in the middle of a zip file
       key = watched_file.sincedb_key
-      begin
-        file_stream = FileInputStream.new(watched_file.path)
-        gzip_stream = GZIPInputStream.new(file_stream)
-        decoder = InputStreamReader.new(gzip_stream, "UTF-8")
-        buffered = BufferedReader.new(decoder)
-        while (line = buffered.readLine(false))
-          watched_file.listener.accept(line)
-          # can't quit, if we did then we would incorrectly write a 'completed' sincedb entry
-          # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading)
-          # should we track lines read in the sincedb and
-          # fast forward through the lines until we reach unseen content?
-          # meaning that we can quit in the middle of a zip file
-        end
-        watched_file.listener.eof
-      rescue ZipException => e
-        logger.error("Cannot decompress the gzip file at path: #{watched_file.path}")
-        watched_file.listener.error
-      else
-        sincedb_collection.store_last_read(key, watched_file.last_stat_size)
-        sincedb_collection.request_disk_flush
-        watched_file.listener.deleted
+
+      if @settings.check_archive_validity && corrupted?(watched_file)
         watched_file.unwatch
-      ensure
-        # rescue each close individually so all close attempts are tried
-        close_and_ignore_ioexception(buffered) unless buffered.nil?
-        close_and_ignore_ioexception(decoder) unless decoder.nil?
-        close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil?
-        close_and_ignore_ioexception(file_stream) unless file_stream.nil?
+      else
+        begin
+          file_stream = FileInputStream.new(watched_file.path)
+          gzip_stream = GZIPInputStream.new(file_stream)
+          decoder = InputStreamReader.new(gzip_stream, "UTF-8")
+          buffered = BufferedReader.new(decoder)
+          while (line = buffered.readLine(false))
+            watched_file.listener.accept(line)
+            # can't quit, if we did then we would incorrectly write a 'completed' sincedb entry
+            # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading)
+            # should we track lines read in the sincedb and
+            # fast forward through the lines until we reach unseen content?
+            # meaning that we can quit in the middle of a zip file
+          end
+          watched_file.listener.eof
+        rescue ZipException => e
+          logger.error("Cannot decompress the gzip file at path: #{watched_file.path}", :exception => e.class,
+                       :message => e.message, :backtrace => e.backtrace)
+          watched_file.listener.error
+        else
+          sincedb_collection.store_last_read(key, watched_file.last_stat_size)
+          sincedb_collection.request_disk_flush
+          watched_file.listener.deleted
+          watched_file.unwatch
+        ensure
+          # rescue each close individually so all close attempts are tried
+          close_and_ignore_ioexception(buffered) unless buffered.nil?
+          close_and_ignore_ioexception(decoder) unless decoder.nil?
+          close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil?
+          close_and_ignore_ioexception(file_stream) unless file_stream.nil?
+        end
       end
       sincedb_collection.clear_watched_file(key)
     end
@@ -56,7 +64,28 @@ module FileWatch module ReadMode module Handlers
       begin
         closeable.close
       rescue Exception => e # IOException can be thrown by any of the Java classes that implement the Closable interface.
-        logger.warn("Ignoring an IOException when closing an instance of #{closeable.class.name}", "exception" => e)
+        logger.warn("Ignoring an IOException when closing an instance of #{closeable.class.name}",
+                    :exception => e.class, :message => e.message, :backtrace => e.backtrace)
+      end
+    end
+
+    def corrupted?(watched_file)
+      begin
+        file_stream = FileInputStream.new(watched_file.path)
+        gzip_stream = GZIPInputStream.new(file_stream)
+        buffer = Java::byte[8192].new
+        start = Time.new
+        until gzip_stream.read(buffer) == -1
+        end
+        return false
+      rescue ZipException => e
+        duration = Time.now - start
+        logger.warn("Detected corrupted archive #{watched_file.path} file won't be processed", :message => e.message,
+                    :duration => duration.round(3))
+        return true
+      ensure
+        close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil?
+        close_and_ignore_ioexception(file_stream) unless file_stream.nil?
       end
     end
   end

data/lib/filewatch/read_mode/processor.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
-require "logstash/util/loggable"
-
+require 'filewatch/processor'
 require_relative "handlers/base"
 require_relative "handlers/read_file"
 require_relative "handlers/read_zip_file"
@@ -9,20 +8,7 @@ module FileWatch module ReadMode
   # Must handle
   #   :read_file
   #   :read_zip_file
-  class Processor
-    include LogStash::Util::Loggable
-
-    attr_reader :watch, :deletable_filepaths
-
-    def initialize(settings)
-      @settings = settings
-      @deletable_filepaths = []
-    end
-
-    def add_watch(watch)
-      @watch = watch
-      self
-    end
+  class Processor < FileWatch::Processor
 
     def initialize_handlers(sincedb_collection, observer)
       # we deviate from the tail mode handler initialization here
@@ -48,24 +34,23 @@ module FileWatch module ReadMode
     private
 
     def process_watched(watched_files)
-      logger.trace("Watched processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the watched state.
       # for a slice of them:
       #   move to the active state
       #   should never have been active before
       # how much of the max active window is available
-      to_take = @settings.max_active - watched_files.count{|wf| wf.active?}
+      to_take = @settings.max_active - watched_files.count { |wf| wf.active? }
       if to_take > 0
-        watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file|
-          path = watched_file.path
+        watched_files.select(&:watched?).take(to_take).each do |watched_file|
           begin
-            watched_file.restat
+            restat(watched_file)
             watched_file.activate
           rescue Errno::ENOENT
-            common_deleted_reaction(watched_file, "Watched")
+            common_deleted_reaction(watched_file, __method__)
             next
           rescue => e
-            common_error_reaction(path, e, "Watched")
+            common_error_reaction(watched_file, e, __method__)
             next
           end
           break if watch.quit?
@@ -74,7 +59,7 @@ module FileWatch module ReadMode
         now = Time.now.to_i
         if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL
           waiting = watched_files.size - @settings.max_active
-          logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}")
+          logger.warn("#{@settings.max_warn_msg}, files yet to open: #{waiting}")
           watch.lastwarn_max_files = now
         end
       end
@@ -83,17 +68,18 @@ module FileWatch module ReadMode
     ## TODO add process_rotation_in_progress
 
     def process_active(watched_files)
-      logger.trace("Active processing")
+      logger.trace(__method__.to_s)
       # Handles watched_files in the active state.
-      watched_files.select {|wf| wf.active? }.each do |watched_file|
-        path = watched_file.path
+      watched_files.each do |watched_file|
+        next unless watched_file.active?
+
         begin
-          watched_file.restat
+          restat(watched_file)
         rescue Errno::ENOENT
-          common_deleted_reaction(watched_file, "Active")
+          common_deleted_reaction(watched_file, __method__)
           next
         rescue => e
-          common_error_reaction(path, e, "Active")
+          common_error_reaction(watched_file, e, __method__)
           next
         end
         break if watch.quit?
@@ -114,19 +100,19 @@ module FileWatch module ReadMode
     def common_detach_when_allread(watched_file)
       watched_file.unwatch
       watched_file.listener.reading_completed
-      deletable_filepaths << watched_file.path
-      logger.trace("Whole file read: #{watched_file.path}, removing from collection")
+      add_deletable_path watched_file.path
+      logger.trace? && logger.trace("whole file read, removing from collection", :path => watched_file.path)
     end
 
     def common_deleted_reaction(watched_file, action)
       # file has gone away or we can't read it anymore.
       watched_file.unwatch
-      deletable_filepaths << watched_file.path
-      logger.trace("#{action} - stat failed: #{watched_file.path}, removing from collection")
+      add_deletable_path watched_file.path
+      logger.trace? && logger.trace("#{action} - stat failed, removing from collection", :path => watched_file.path)
     end
 
-    def common_error_reaction(path, error, action)
-      logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})")
+    def common_error_reaction(watched_file, error, action)
+      logger.error("#{action} - other error", error_details(error, watched_file))
     end
   end
 end end

data/lib/filewatch/settings.rb

@@ -6,9 +6,10 @@ module FileWatch
     attr_reader :max_active, :max_warn_msg, :lastwarn_max_files
     attr_reader :sincedb_write_interval, :stat_interval, :discover_interval
     attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size
-    attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration
+    attr_reader :sincedb_path, :sincedb_expiry_duration
     attr_reader :file_sort_by, :file_sort_direction
     attr_reader :exit_after_read
+    attr_reader :check_archive_validity
 
     def self.from_options(opts)
       new.add_options(opts)
@@ -40,7 +41,6 @@ module FileWatch
       @file_chunk_size = @opts[:file_chunk_size]
       @close_older = @opts[:close_older]
       @ignore_older = @opts[:ignore_older]
-      @sincedb_write_interval = @opts[:sincedb_write_interval]
       @stat_interval = @opts[:stat_interval]
       @discover_interval = @opts[:discover_interval]
       @exclude = Array(@opts[:exclude])
@@ -52,6 +52,7 @@ module FileWatch
       @file_sort_by = @opts[:file_sort_by]
       @file_sort_direction = @opts[:file_sort_direction]
       @exit_after_read = @opts[:exit_after_read]
+      @check_archive_validity = @opts[:check_archive_validity]
       self
     end
 

data/lib/filewatch/sincedb_collection.rb

@@ -56,12 +56,12 @@ module FileWatch
         logger.trace("open: count of keys read: #{@sincedb.keys.size}")
       rescue => e
         #No existing sincedb to load
-        logger.trace("open: error: #{path}: #{e.inspect}")
+        logger.trace("open: error:", :path => path, :exception => e.class, :message => e.message)
       end
     end
 
     def associate(watched_file)
-      logger.trace("associate: finding", "inode" => watched_file.sincedb_key.inode, "path" => watched_file.path)
+      logger.trace? && logger.trace("associate: finding", :path => watched_file.path, :inode => watched_file.sincedb_key.inode)
       sincedb_value = find(watched_file)
       if sincedb_value.nil?
         # sincedb has no record of this inode
@@ -71,7 +71,8 @@ module FileWatch
         logger.trace("associate: unmatched")
         return true
       end
-      logger.trace("associate: found sincedb record", "filename" => watched_file.filename, "sincedb key" => watched_file.sincedb_key,"sincedb_value" => sincedb_value)
+      logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename,
+                                    :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
       if sincedb_value.watched_file.nil?
         # not associated
         if sincedb_value.path_in_sincedb.nil?
@@ -106,7 +107,8 @@ module FileWatch
       #   after the original is deleted
       # are not yet in the delete phase, let this play out
       existing_watched_file = sincedb_value.watched_file
-      logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename", "this watched_file details" => watched_file.details, "other watched_file details" => existing_watched_file.details)
+      logger.trace? && logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename",
+                                    :this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details)
       watched_file.rotation_in_progress
       true
     end
@@ -149,8 +151,8 @@ module FileWatch
     end
 
     def watched_file_deleted(watched_file)
-      return unless member?(watched_file.sincedb_key)
-      get(watched_file.sincedb_key).unset_watched_file
+      value = @sincedb[watched_file.sincedb_key]
+      value.unset_watched_file if value
     end
 
     def store_last_read(key, pos)
@@ -178,24 +180,24 @@ module FileWatch
       get(key).watched_file.nil?
     end
 
-    private
-
     def flush_at_interval
-      now = Time.now.to_i
-      delta = now - @sincedb_last_write
+      now = Time.now
+      delta = now.to_i - @sincedb_last_write
       if delta >= @settings.sincedb_write_interval
         logger.debug("writing sincedb (delta since last write = #{delta})")
         sincedb_write(now)
       end
     end
 
+    private
+
     def handle_association(sincedb_value, watched_file)
       watched_file.update_bytes_read(sincedb_value.position)
       sincedb_value.set_watched_file(watched_file)
       watched_file.initial_completed
       if watched_file.all_read?
         watched_file.ignore
-        logger.trace("handle_association fully read, ignoring.....", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
+        logger.trace? && logger.trace("handle_association fully read, ignoring.....", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
       end
     end
 
@@ -208,33 +210,33 @@ module FileWatch
       end
     end
 
-    def sincedb_write(time = Time.now.to_i)
-      logger.trace("sincedb_write: to: #{path}")
+    def sincedb_write(time = Time.now)
+      logger.trace("sincedb_write: #{path} (time = #{time})")
       begin
-        @write_method.call
+        @write_method.call(time)
         @serializer.expired_keys.each do |key|
           @sincedb[key].unset_watched_file
           delete(key)
-          logger.trace("sincedb_write: cleaned", "key" => "'#{key}'")
+          logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
         end
-        @sincedb_last_write = time
+        @sincedb_last_write = time.to_i
         @write_requested = false
       rescue Errno::EACCES
         # no file handles free perhaps
         # maybe it will work next time
-        logger.trace("sincedb_write: error: #{path}: #{$!}")
+        logger.trace("sincedb_write: #{path} error: #{$!}")
       end
     end
 
-    def atomic_write
+    def atomic_write(time)
       FileHelper.write_atomically(@full_path) do |io|
-        @serializer.serialize(@sincedb, io)
+        @serializer.serialize(@sincedb, io, time.to_f)
       end
     end
 
-    def non_atomic_write
+    def non_atomic_write(time)
       IO.open(IO.sysopen(@full_path, "w+")) do |io|
-        @serializer.serialize(@sincedb, io)
+        @serializer.serialize(@sincedb, io, time.to_f)
       end
     end
   end

data/lib/filewatch/stat/generic.rb

@@ -3,24 +3,19 @@
 module FileWatch module Stat
   class Generic
 
-    attr_reader :identifier, :inode, :modified_at, :size, :inode_struct
+    attr_reader :inode, :modified_at, :size, :inode_struct
 
     def initialize(source)
-      @source = source
-      @identifier = nil
+      @source = source # Pathname
       restat
     end
 
-    def add_identifier(identifier) self; end
-
     def restat
-      @inner_stat = @source.stat
-      @inode = @inner_stat.ino.to_s
-      @modified_at = @inner_stat.mtime.to_f
-      @size = @inner_stat.size
-      @dev_major = @inner_stat.dev_major
-      @dev_minor = @inner_stat.dev_minor
-      @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor)
+      stat = @source.stat
+      @inode = stat.ino.to_s
+      @modified_at = stat.mtime.to_f
+      @size = stat.size
+      @inode_struct = InodeStruct.new(@inode, stat.dev_major, stat.dev_minor)
     end
 
     def windows?
@@ -28,7 +23,7 @@ module FileWatch module Stat
     end
 
     def inspect
-      "<Generic size='#{@size}', modified_at='#{@modified_at}', inode='#{@inode}', inode_struct='#{@inode_struct}'>"
+      "<#{self.class.name} size=#{@size}, modified_at=#{@modified_at}, inode='#{@inode}', inode_struct=#{@inode_struct}>"
     end
   end
 end end