filewatch 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 846e8322c5eaff65f07c8fc29a38252fc012848b
-  data.tar.gz: 2d6f2d06f3fce3016decdeb4d7f19c2fc033d63b
+  metadata.gz: 59ac6ba9efca68da9ea0b3342e87deab3635f53c
+  data.tar.gz: 5a71f0673de75e1444584b01c0dcfe56e039d2ce
 SHA512:
-  metadata.gz: a6ddbce50ce399c9201f60c54790bf310afb0658f577ef6ae0de16f552f9c4c2aab70106eb34f06f5d8eadbcc492743bd03232030e2e39363b75bcc9988c4c1e
-  data.tar.gz: eec3de133bc8998d9f956655a07c63725b0c69a111a93355740b1e615b7615ed3c980dc048de7927e3138d455420b6b636fbfbc04e124c26a05eb7f41d974f18
+  metadata.gz: 366003c080654805fa4da59a6ef354a25cb0a8495090d668ad0dd8ae628e7bf7438b5d0894938925bed8d8e2b08a6dae3671a0772480d94d210039273aaa7e01
+  data.tar.gz: d05f46ef57830653fbd19c3eef0d4e49fd4137f5604d0296df1991fc64ae0b6b74c926a7e2796bf528d4c55d1571f8a9b190115b88571110027bb16ebe6fe8fb

data/lib/filewatch/observing_tail.rb

@@ -21,43 +21,45 @@ module FileWatch
 
     def subscribe(observer = NullObserver.new)
       @watch.subscribe(@opts[:stat_interval],
-                       @opts[:discover_interval]) do |event, path|
+                       @opts[:discover_interval]) do |event, watched_file|
+        path = watched_file.path
+        file_is_open = watched_file.file_open?
         listener = observer.listener_for(path)
         case event
+        when :unignore
+          listener.created
+          _add_to_sincedb(watched_file, event) unless @sincedb.member?(watched_file.inode)
         when :create, :create_initial
-          if @files.member?(path)
-            @logger.debug? && @logger.debug("#{event} for #{path}: already exists in @files")
+          if file_is_open
+            @logger.debug? && @logger.debug("#{event} for #{path}: file already open")
             next
           end
-          if _open_file(path, event)
+          if _open_file(watched_file, event)
             listener.created
-            observe_read_file(path, listener)
+            observe_read_file(watched_file, listener)
           end
         when :modify
-          if !@files.member?(path)
-            @logger.debug? && @logger.debug(":modify for #{path}, does not exist in @files")
-            if _open_file(path, event)
-              observe_read_file(path, listener)
-            end
+          if file_is_open
+            observe_read_file(watched_file, listener)
           else
-            observe_read_file(path, listener)
+            @logger.debug? && @logger.debug(":modify for #{path}, file is not open, opening now")
+            if _open_file(watched_file, event)
+              observe_read_file(watched_file, listener)
+            end
           end
         when :delete
-          @logger.debug? && @logger.debug(":delete for #{path}, deleted from @files")
-          if @files[path]
-            observe_read_file(path, listener)
-            @files[path].close
+          if file_is_open
+            @logger.debug? && @logger.debug(":delete for #{path}, closing file")
+            observe_read_file(watched_file, listener)
+            watched_file.file_close
+          else
+            @logger.debug? && @logger.debug(":delete for #{path}, file already closed")
           end
           listener.deleted
-          @files.delete(path)
-          @statcache.delete(path)
         when :timeout
-          @logger.debug? && @logger.debug(":timeout for #{path}, deleted from @files")
-          if (deleted = @files.delete(path))
-            deleted.close
-          end
+          @logger.debug? && @logger.debug(":timeout for #{path}, closing file")
+          watched_file.file_close
           listener.timed_out
-          @statcache.delete(path)
         else
           @logger.warn("unknown event type #{event} for #{path}")
         end
@@ -65,24 +67,29 @@ module FileWatch
     end # def subscribe
 
     private
-    def observe_read_file(path, listener)
-      @buffers[path] ||= FileWatch::BufferedTokenizer.new(@opts[:delimiter])
-      delimiter_byte_size = @opts[:delimiter].bytesize
+    def observe_read_file(watched_file, listener)
       changed = false
       loop do
         begin
-          data = @files[path].sysread(32768)
+          data = watched_file.file_read(32768)
           changed = true
-          @buffers[path].extract(data).each do |line|
+          watched_file.buffer_extract(data).each do |line|
             listener.accept(line)
-            @sincedb[@statcache[path]] += (line.bytesize + delimiter_byte_size)
+            @sincedb[watched_file.inode] += (line.bytesize + @delimiter_byte_size)
           end
+          # watched_file bytes_read tracks the sincedb entry
+          # see TODO in watch.rb
+          watched_file.update_bytes_read(@sincedb[watched_file.inode])
         rescue EOFError
           listener.eof
           break
         rescue Errno::EWOULDBLOCK, Errno::EINTR
           listener.error
           break
+        rescue => e
+          @logger.debug? && @logger.debug("observe_read_file: general error reading #{watched_file.path} - error: #{e.inspect}")
+          listener.error
+          break
         end
       end
 

data/lib/filewatch/tail_base.rb

@@ -1,5 +1,4 @@
 require "filewatch/helper"
-require "filewatch/buftok"
 require "filewatch/watch"
 
 if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
@@ -14,14 +13,15 @@ require "JRubyFileExtension.jar" if defined? JRUBY_VERSION
 module FileWatch
   module TailBase
     # how often (in seconds) we @logger.warn a failed file open, per path.
-    OPEN_WARN_INTERVAL = ENV["FILEWATCH_OPEN_WARN_INTERVAL"] ?
-                         ENV["FILEWATCH_OPEN_WARN_INTERVAL"].to_i : 300
+    OPEN_WARN_INTERVAL = ENV.fetch("FILEWATCH_OPEN_WARN_INTERVAL", 300).to_i
 
     attr_reader :logger
 
     class NoSinceDBPathGiven < StandardError; end
 
     public
+    # TODO move sincedb to watch.rb
+    # see TODO there
     def initialize(opts={})
       @iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
 
@@ -31,14 +31,12 @@ module FileWatch
         @logger = Logger.new(STDERR)
         @logger.level = Logger::INFO
       end
-      @files = {}
       @lastwarn = Hash.new { |h, k| h[k] = 0 }
       @buffers = {}
       @watch = FileWatch::Watch.new
       @watch.logger = @logger
-      @sincedb = {}
       @sincedb_last_write = 0
-      @statcache = {}
+      @sincedb = {}
       @opts = {
         :sincedb_write_interval => 10,
         :stat_interval => 1,
@@ -57,6 +55,9 @@ module FileWatch
       @watch.exclude(@opts[:exclude])
       @watch.close_older = @opts[:close_older]
       @watch.ignore_older = @opts[:ignore_older]
+      @watch.delimiter = @opts[:delimiter]
+      @watch.max_open_files = @opts[:max_open_files]
+      @delimiter_byte_size = @opts[:delimiter].bytesize
 
       _sincedb_open
     end # def initialize
@@ -74,28 +75,20 @@ module FileWatch
 
     public
     def sincedb_record_uid(path, stat)
-      inode = @watch.inode(path,stat)
-      @statcache[path] = inode
-      return inode
+      # retain this call because its part of the public API
+      @watch.inode(path, stat)
     end # def sincedb_record_uid
 
     private
 
-    def file_expired?(stat)
-      return false if @opts[:ignore_older].nil?
-      # (Time.now - stat.mtime) <- in jruby, this does int and float
-      # conversions before the subtraction and returns a float.
-      # so use all ints instead
-      (Time.now.to_i - stat.mtime.to_i) > @opts[:ignore_older]
-    end
-
-    def _open_file(path, event)
+    def _open_file(watched_file, event)
+      path = watched_file.path
       @logger.debug? && @logger.debug("_open_file: #{path}: opening")
       begin
         if @iswindows && defined? JRUBY_VERSION
-          @files[path] = Java::RubyFileExt::getRubyFile(path)
+          watched_file.file_add_opened(Java::RubyFileExt::getRubyFile(path))
         else
-          @files[path] = File.open(path)
+          watched_file.file_add_opened(File.open(path))
         end
       rescue
         # don't emit this message too often. if a file that we can't
@@ -106,46 +99,68 @@ module FileWatch
           @logger.warn("failed to open #{path}: #{$!}")
           @lastwarn[path] = now
         else
-          @logger.debug? && @logger.debug("(warn supressed) failed to open #{path}: #{$!}")
+          @logger.debug? && @logger.debug("(warn supressed) failed to open #{path}: #{$!.inspect}")
         end
-        @files.delete(path)
+        watched_file.watch # set it back to watch so we can try it again
         return false
       end
+      _add_to_sincedb(watched_file, event)
+      true
+    end # def _open_file
 
-      stat = File::Stat.new(path)
-      sincedb_record_uid = sincedb_record_uid(path, stat)
-
-      expired_based_size = file_expired?(stat) ? stat.size : 0
-
-      if @sincedb.member?(sincedb_record_uid)
-        last_size = @sincedb[sincedb_record_uid]
-        @logger.debug? && @logger.debug("#{path}: sincedb last value #{@sincedb[sincedb_record_uid]}, cur size #{stat.size}")
-        if last_size <= stat.size
-          @logger.debug? && @logger.debug("#{path}: sincedb: seeking to #{last_size}")
-          @files[path].sysseek(last_size, IO::SEEK_SET)
+    def _add_to_sincedb(watched_file, event)
+      # called when newly discovered files are opened
+      stat = watched_file.filestat
+      sincedb_key = watched_file.inode
+      path = watched_file.path
+
+      if @sincedb.member?(sincedb_key)
+        # we have seen this inode before
+        # but this is a new watched_file
+        # and we can't tell if its contents are the same
+        # as another file we have watched before.
+        last_read_size = @sincedb[sincedb_key]
+        @logger.debug? && @logger.debug("#{path}: sincedb last value #{@sincedb[sincedb_key]}, cur size #{stat.size}")
+        if stat.size > last_read_size
+          # 1) it could really be a new file with lots of new content
+          # 2) it could have old content that was read plus new that is not
+          @logger.debug? && @logger.debug("#{path}: sincedb: seeking to #{last_read_size}")
+          watched_file.file_seek(last_read_size) # going with 2
+          watched_file.update_bytes_read(last_read_size)
+        elsif stat.size == last_read_size
+          # 1) it could have old content that was read
+          # 2) it could have new content that happens to be the same size
+          @logger.debug? && @logger.debug("#{path}: sincedb: seeking to #{last_read_size}")
+          watched_file.file_seek(last_read_size) # going with 1.
+          watched_file.update_bytes_read(last_read_size)
         else
+          # it seems to be a new file with less content
           @logger.debug? && @logger.debug("#{path}: last value size is greater than current value, starting over")
-          @sincedb[sincedb_record_uid] = 0
+          @sincedb[sincedb_key] = 0
+          watched_file.update_bytes_read(0) if watched_file.bytes_read != 0
         end
-      elsif event == :create_initial && @files[path]
+      elsif event == :create_initial
         if @opts[:start_new_files_at] == :beginning
           @logger.debug? && @logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
-          @files[path].sysseek(expired_based_size, IO::SEEK_SET)
-          @sincedb[sincedb_record_uid] = expired_based_size
+          watched_file.file_seek(0)
+          @sincedb[sincedb_key] = 0
         else
           # seek to end
           @logger.debug? && @logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
-          @files[path].sysseek(stat.size, IO::SEEK_SET)
-          @sincedb[sincedb_record_uid] = stat.size
+          watched_file.file_seek(stat.size)
+          @sincedb[sincedb_key] = stat.size
         end
-      elsif event == :create && @files[path]
-        @sincedb[sincedb_record_uid] = expired_based_size
+      elsif event == :create
+        @sincedb[sincedb_key] = 0
+      elsif event == :modify && @sincedb[sincedb_key].nil?
+        @sincedb[sincedb_key] = 0
+      elsif event == :unignore
+        @sincedb[sincedb_key] = watched_file.bytes_read
       else
         @logger.debug? && @logger.debug("#{path}: staying at position 0, no sincedb")
       end
-
       return true
-    end # def _open_file
+    end # def _add_to_sincedb
 
     public
     def sincedb_write(reason=nil)
@@ -157,30 +172,34 @@ module FileWatch
     def _sincedb_open
       path = @opts[:sincedb_path]
       begin
-        db = File.open(path)
+        File.open(path) do |db|
+          @logger.debug? && @logger.debug("_sincedb_open: reading from #{path}")
+          db.each do |line|
+            ino, dev_major, dev_minor, pos = line.split(" ", 4)
+            sincedb_key = [ino, dev_major.to_i, dev_minor.to_i]
+            @logger.debug? && @logger.debug("_sincedb_open: setting #{sincedb_key.inspect} to #{pos.to_i}")
+            @sincedb[sincedb_key] = pos.to_i
+          end
+        end
       rescue
         #No existing sincedb to load
-        @logger.debug? && @logger.debug("_sincedb_open: #{path}: #{$!}")
-        return
-      end
-
-      @logger.debug? && @logger.debug("_sincedb_open: reading from #{path}")
-      db.each do |line|
-        ino, dev_major, dev_minor, pos = line.split(" ", 4)
-        sincedb_record_uid = [ino, dev_major.to_i, dev_minor.to_i]
-        @logger.debug? && @logger.debug("_sincedb_open: setting #{sincedb_record_uid.inspect} to #{pos.to_i}")
-        @sincedb[sincedb_record_uid] = pos.to_i
+        @logger.debug? && @logger.debug("_sincedb_open: error: #{path}: #{$!}")
       end
-      db.close
     end # def _sincedb_open
 
     private
     def _sincedb_write
       path = @opts[:sincedb_path]
-      if @iswindows || File.device?(path)
-        IO.write(path, serialize_sincedb, 0)
-      else
-        File.atomic_write(path) {|file| file.write(serialize_sincedb) }
+      begin
+        if @iswindows || File.device?(path)
+          IO.write(path, serialize_sincedb, 0)
+        else
+          File.atomic_write(path) {|file| file.write(serialize_sincedb) }
+        end
+      rescue Errno::EACCES
+        # probably no file handles free
+        # maybe it will work next time
+        @logger.debug? && @logger.debug("_sincedb_write: error: #{path}: #{$!}")
       end
     end # def _sincedb_write
 
@@ -189,13 +208,13 @@ module FileWatch
     # it should be called for clean up
     # before the instance is disposed of.
     def quit
+      @watch.quit # <-- should close all the files
+      # and that should allow the sincedb_write to succeed if it could not before
       _sincedb_write
-      @watch.quit
-      @files.each {|path, file| file.close }
-      @files.clear
     end # def quit
 
     public
+
     # close_file(path) is to be used by external code
     # when it knows that it is completely done with a file.
     # Other files or folders may still be being watched.
@@ -204,10 +223,7 @@ module FileWatch
     # The sysadmin should rename, move or delete the file.
     def close_file(path)
       @watch.unwatch(path)
-      file = @files.delete(path)
-      return if file.nil?
       _sincedb_write
-      file.close
     end
 
     private

data/lib/filewatch/watch.rb

@@ -1,59 +1,42 @@
 require "logger"
+require_relative 'watched_file'
+
 if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
   require "filewatch/winhelper"
+  FILEWATCH_INODE_METHOD = :win_inode
+else
+  FILEWATCH_INODE_METHOD = :nix_inode
 end
 
 module FileWatch
+  # TODO make a WatchedFilesDb class that holds the watched_files instead of a hash
+  # it should support an 'identity' of path + inode
+  # it should be serializable instead of the sincedb
+  # it should be deserializable to recreate the exact state all files were in as last seen
+  # some parts of the each method should be handled by it, e.g.
+  # wfs_db.<state>_iterator{|wf| }, trapping the Errno::ENOENT, auto_delete and yield wtached_file
   class Watch
-    class WatchedFile
-      def self.new_initial(path, inode)
-        new(path, inode, true)
-      end
-
-      def self.new_ongoing(path, inode)
-        new(path, inode, false)
-      end
-
-      attr_reader :size, :inode
-      attr_writer :create_sent, :initial, :timeout_sent
-
-      attr_reader :path
-
-      def initialize(path, inode, initial)
-        @path = path
-        @size, @create_sent, @timeout_sent = 0, false, false
-        @inode, @initial = inode, initial
-      end
-
-      def update(stat, inode = nil)
-        @size = stat.size
-        @inode = inode if inode
-      end
-
-      def clear_timeout
-        @timeout_sent = false
-      end
 
-      def create_sent?
-        @create_sent
-      end
+    MAX_FILES_WARN_INTERVAL = ENV.fetch("FILEWATCH_MAX_FILES_WARN_INTERVAL", 20).to_i
 
-      def initial?
-        @initial
-      end
+    def self.win_inode(path, stat)
+      fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
+      [fileId, 0, 0] # dev_* doesn't make sense on Windows
+    end
 
-      def timeout_sent?
-        @timeout_sent
-      end
+    def self.nix_inode(path, stat)
+      [stat.ino.to_s, stat.dev_major, stat.dev_minor]
+    end
 
-      def to_s() inspect; end
+    def self.inode(path, stat)
+      send(FILEWATCH_INODE_METHOD, path, stat)
     end
 
-    attr_accessor :logger, :close_older, :ignore_older
+    attr_accessor :logger
+    attr_accessor :delimiter
+    attr_reader :max_active
 
-    public
     def initialize(opts={})
-      @iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
       if opts[:logger]
         @logger = opts[:logger]
       else
@@ -62,62 +45,65 @@ module FileWatch
       end
       @watching = []
       @exclude = []
-      @files = Hash.new { |h, k| h[k] = WatchedFile.new(k, false, false) }
-      @unwatched = Hash.new
+      @files = Hash.new
       # we need to be threadsafe about the mutation
       # of the above 2 ivars because the public
-      # methods each, discover, watch and unwatch
+      # methods each, discover and watch
       # can be called from different threads.
       @lock = Mutex.new
       # we need to be threadsafe about the quit mutation
       @quit = false
       @quit_lock = Mutex.new
+      self.max_open_files = ENV["FILEWATCH_MAX_OPEN_FILES"].to_i
+      @lastwarn_max_files = 0
     end # def initialize
 
     public
+
+    def max_open_files=(value)
+      val = value.to_i
+      val = 4095 if value.nil? || val <= 0
+      @max_warn_msg = "Reached open files limit: #{val}, set by the 'max_open_files' option or default"
+      @max_active = val
+    end
+
+    def ignore_older=(value)
+      #nil is allowed but 0 and negatives are made nil
+      if !value.nil?
+        val = value.to_f
+        val = val <= 0 ? nil : val
+      end
+      @ignore_older = val
+    end
+
+    def close_older=(value)
+      if !value.nil?
+        val = value.to_f
+        val = val <= 0 ? nil : val
+      end
+      @close_older = val
+    end
+
     def exclude(path)
       path.to_a.each { |p| @exclude << p }
     end
 
-    public
     def watch(path)
       synchronized do
         if !@watching.member?(path)
           @watching << path
           _discover_file(path) do |filepath, stat|
-            WatchedFile.new_initial(filepath, inode(filepath, stat))
+            WatchedFile.new_initial(
+                filepath, inode(filepath, stat), stat
+              ).init_vars(@delimiter, @ignore_older, @close_older)
           end
         end
       end
       return true
     end # def watch
 
-    def unwatch(path)
-      synchronized do
-        result = false
-        if @watching.delete(path)
-          _globbed_files(path).each do |file|
-            deleted = @files.delete(file)
-            @unwatched[file] = deleted if deleted
-          end
-          result = true
-        else
-          result = @files.delete(path)
-          @unwatched[path] = result if result
-        end
-        return !!result
-      end
-    end
-
-    public
-    def inode(path,stat)
-      if @iswindows
-        fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
-        inode = [fileId, 0, 0] # dev_* doesn't make sense on Windows
-      else
-        inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
-      end
-      return inode
+    def inode(path, stat)
+      self.class.inode(path, stat)
     end
 
     # Calls &block with params [event_type, path]
@@ -126,77 +112,147 @@ module FileWatch
     #   :create - file is created (new file after initial globs, start at 0)
     #   :modify - file is modified (size increases)
     #   :delete - file is deleted
-    public
+    #   :timeout - file is closable
+    #   :unignore - file was ignored, but since then it received new content
     def each(&block)
       synchronized do
+        return if @files.empty?
+
+        file_deleteable = []
+        # creates this array just once
+        watched_files = @files.values
+
+        # look at the closed to see if its changed
+        watched_files.select {|wf| wf.closed? }.each do |watched_file|
+          path = watched_file.path
+          begin
+            stat = watched_file.restat
+            if watched_file.size_changed? || watched_file.inode_changed?(inode(path,stat))
+              # if the closed file changed, move it to the watched state
+              # not to active state because we want to use MAX_OPEN_FILES throttling.
+              watched_file.watch
+            end
+          rescue Errno::ENOENT
+            # file has gone away or we can't read it anymore.
+            file_deleteable << path
+            @logger.debug? && @logger.debug("each: closed: stat failed: #{path}: (#{$!}), deleting from @files")
+          rescue => e
+            @logger.debug? && @logger.debug("each: closed: stat failed: #{path}: (#{e.inspect})")
+          end
+        end
+
+        # look at the ignored to see if its changed
+        watched_files.select {|wf| wf.ignored? }.each do |watched_file|
+          path = watched_file.path
+          begin
+            stat = watched_file.restat
+            if watched_file.size_changed? || watched_file.inode_changed?(inode(path,stat))
+              # if the ignored file changed, move it to the watched state
+              # not to active state because we want to use MAX_OPEN_FILES throttling.
+              # this file has not been yielded to the block yet
+              # but we must have the tail to start from the end, so when the file
+              # was first ignored we updated the bytes_read to the stat.size at that time.
+              # by adding this to the sincedb so that the subsequent modify
+              # event can detect the change
+              watched_file.watch
+              yield(:unignore, watched_file)
+            end
+          rescue Errno::ENOENT
+            # file has gone away or we can't read it anymore.
+            file_deleteable << path
+            @logger.debug? && @logger.debug("each: ignored: stat failed: #{path}: (#{$!}), deleting from @files")
+          rescue => e
+            @logger.debug? && @logger.debug("each: ignored: stat failed: #{path}: (#{e.inspect})")
+          end
+        end
+
         # Send any creates.
-        @files.each do |path, watched_file|
-          if !watched_file.create_sent?
+        if (to_take = @max_active - watched_files.count{|wf| wf.active?}) > 0
+          watched_files.select {|wf| wf.watched? }.take(to_take).each do |watched_file|
+            watched_file.activate
+            # don't do create again
+            next if watched_file.state_history_any?(:closed, :ignored)
+            # if the file can't be opened during the yield
+            # its state is set back to watched
             if watched_file.initial?
-              yield(:create_initial, path)
+              yield(:create_initial, watched_file)
+            else
+              yield(:create, watched_file)
+            end
+          end
+        else
+          now = Time.now.to_i
+          if (now - @lastwarn_max_files) > MAX_FILES_WARN_INTERVAL
+            waiting = @files.size - @max_active
+            specific = if @close_older.nil?
+             ", try setting close_older. There are #{waiting} unopened files"
             else
-              yield(:create, path)
+              ", files yet to open: #{waiting}"
             end
-            watched_file.create_sent = true
+            @logger.warn(@max_warn_msg + specific)
+            @lastwarn_max_files = now
           end
         end
 
-        @files.each do |path, watched_file|
+        # wf.active means the actual files were opened
+        # and have been read once - unless they were empty at the time
+        watched_files.select {|wf| wf.active? }.each do |watched_file|
+          path = watched_file.path
           begin
-            stat = File::Stat.new(path)
+            stat = watched_file.restat
           rescue Errno::ENOENT
             # file has gone away or we can't read it anymore.
-            @files.delete(path)
-            @logger.debug? && @logger.debug("#{path}: stat failed (#{$!}), deleting from @files")
-            yield(:delete, path)
+            file_deleteable << path
+            @logger.debug? && @logger.debug("each: active: stat failed: #{path}: (#{$!}), deleting from @files")
+            watched_file.unwatch
+            yield(:delete, watched_file)
+            next
+          rescue => e
+            @logger.debug? && @logger.debug("each: active: stat failed: #{path}: (#{e.inspect})")
             next
           end
 
-          if file_closable?(stat, watched_file)
-            if !watched_file.timeout_sent?
-              @logger.debug? && @logger.debug("#{path}: file expired")
-              yield(:timeout, path)
-              watched_file.timeout_sent = true
-            end
+          if watched_file.file_closable?
+            @logger.debug? && @logger.debug("each: active: file expired: #{path}")
+            yield(:timeout, watched_file)
+            watched_file.close
             next
           end
 
-          inode = inode(path,stat)
-          old_size = watched_file.size
-
-          if inode != watched_file.inode
-            @logger.debug? && @logger.debug("#{path}: old inode was #{watched_file.inode.inspect}, new is #{inode.inspect}")
-            yield(:delete, path)
-            yield(:create, path)
-            watched_file.update(stat, inode)
-          elsif stat.size < old_size
-            @logger.debug? && @logger.debug("#{path}: file rolled, new size is #{stat.size}, old size #{old_size}")
-            yield(:delete, path)
-            yield(:create, path)
-            watched_file.update(stat, inode)
-          elsif stat.size > old_size
-            @logger.debug? && @logger.debug("#{path}: file grew, old size #{old_size}, new size #{stat.size}")
-            yield(:modify, path)
-            # if there is a material change to the file, re-enable timeout
-            watched_file.clear_timeout
-            watched_file.update(stat, inode)
+          _inode = inode(path,stat)
+          read_thus_far = watched_file.bytes_read
+          # we don't update the size here, its updated when we actually read
+          if watched_file.inode_changed?(_inode)
+            @logger.debug? && @logger.debug("each: new inode: #{path}: old inode was #{watched_file.inode.inspect}, new is #{_inode.inspect}")
+            watched_file.update_inode(_inode)
+            yield(:delete, watched_file)
+            yield(:create, watched_file)
+          elsif stat.size < read_thus_far
+            @logger.debug? && @logger.debug("each: file rolled: #{path}: new size is #{stat.size}, old size #{read_thus_far}")
+            yield(:delete, watched_file)
+            yield(:create, watched_file)
+          elsif stat.size > read_thus_far
+            @logger.debug? && @logger.debug("each: file grew: #{path}: old size #{read_thus_far}, new size #{stat.size}")
+            yield(:modify, watched_file)
           end
         end
+
+        file_deleteable.each {|f| @files.delete(f)}
       end
     end # def each
 
-    public
     def discover
       synchronized do
         @watching.each do |path|
           _discover_file(path) do |filepath, stat|
-            WatchedFile.new_ongoing(filepath, inode(filepath, stat))
+            WatchedFile.new_ongoing(
+                filepath, inode(filepath, stat), stat
+              ).init_vars(@delimiter, @ignore_older, @close_older)
           end
         end
       end
     end
 
-    public
     def subscribe(stat_interval = 1, discover_interval = 5, &block)
       glob = 0
       reset_quit
@@ -208,80 +264,62 @@ module FileWatch
           discover
           glob = 0
         end
-
+        break if quit?
         sleep(stat_interval)
       end
+      @files.values.each(&:file_close)
     end # def subscribe
 
-    private
-    def file_closable?(stat, watched_file)
-      file_can_close?(stat) && watched_file.size == stat.size
-    end
-
-    def file_ignorable?(stat)
-      return false unless expiry_ignore_enabled?
-      # (Time.now - stat.mtime) <- in jruby, this does int and float
-      # conversions before the subtraction and returns a float.
-      # so use all ints instead
-      (Time.now.to_i - stat.mtime.to_i) > @ignore_older
-    end
+    def quit
+      @quit_lock.synchronize { @quit = true }
+    end # def quit
 
-    def file_can_close?(stat)
-      return false unless expiry_close_enabled?
-      # (Time.now - stat.mtime) <- in jruby, this does int and float
-      # conversions before the subtraction and returns a float.
-      # so use all ints instead
-      (Time.now.to_i - stat.mtime.to_i) > @close_older
+    def quit?
+      @quit_lock.synchronize { @quit }
     end
 
     private
+
     def _discover_file(path)
       _globbed_files(path).each do |file|
-        next if @files.member?(file)
-        next if @unwatched.member?(file)
         next unless File.file?(file)
-
-        @logger.debug? && @logger.debug("_discover_file: #{path}: new: #{file} (exclude is #{@exclude.inspect})")
+        new_discovery = false
+        if @files.member?(file)
+          watched_file = @files[file]
+        else
+          @logger.debug? && @logger.debug("_discover_file: #{path}: new: #{file} (exclude is #{@exclude.inspect})")
+          # let the caller build the object in its context
+          new_discovery = true
+          watched_file = yield(file, File::Stat.new(file))
+        end
 
         skip = false
         @exclude.each do |pattern|
           if File.fnmatch?(pattern, File.basename(file))
             @logger.debug? && @logger.debug("_discover_file: #{file}: skipping because it " +
-                          "matches exclude #{pattern}")
+                          "matches exclude #{pattern}") if new_discovery
             skip = true
+            watched_file.unwatch
             break
           end
         end
         next if skip
 
-        stat = File::Stat.new(file)
-        # let the caller build the object in its context
-        watched_file = yield(file, stat)
-
-        if file_ignorable?(stat)
-          msg = "_discover_file: #{file}: skipping because it was last modified more than #{@ignore_older} seconds ago"
-          @logger.debug? && @logger.debug(msg)
-          # we update the size on discovery here
-          # so the existing contents are not read.
-          # because, normally, a newly discovered file will
-          # have a watched_file size of zero
-          watched_file.update(stat)
+        if new_discovery
+          if watched_file.file_ignorable?
+            @logger.debug? && @logger.debug("_discover_file: #{file}: skipping because it was last modified more than #{@ignore_older} seconds ago")
+            # on discovery we put watched_file into the ignored state and that
+            # updates the size from the internal stat
+            # so the existing contents are not read.
+            # because, normally, a newly discovered file will
+            # have a watched_file size of zero
+            watched_file.ignore
+          end
+          @files[file] = watched_file
         end
-
-        @files[file] = watched_file
       end
     end # def _discover_file
 
-    private
-    def expiry_close_enabled?
-      !@close_older.nil?
-    end
-
-    private
-    def expiry_ignore_enabled?
-      !@ignore_older.nil?
-    end
-
     private
     def _globbed_files(path)
       globbed_dirs = Dir.glob(path)
@@ -299,19 +337,11 @@ module FileWatch
       @lock.synchronize { block.call }
     end
 
-    private
-    def quit?
-      @quit_lock.synchronize { @quit }
-    end
+    public
 
     private
     def reset_quit
       @quit_lock.synchronize { @quit = false }
     end
-
-    public
-    def quit
-      @quit_lock.synchronize { @quit = true }
-    end # def quit
   end # class Watch
 end # module FileWatch

data/lib/filewatch/watched_file.rb

@@ -0,0 +1,173 @@
+require "filewatch/buftok"
+
+module FileWatch
+  class WatchedFile
+    def self.new_initial(path, inode, stat)
+      new(path, inode, stat, true)
+    end
+
+    def self.new_ongoing(path, inode, stat)
+      new(path, inode, stat, false)
+    end
+
+    attr_reader :bytes_read, :inode, :state, :file, :buffer, :state_history
+    attr_reader :path, :filestat, :accessed_at
+    attr_accessor :close_older, :ignore_older, :delimiter
+
+    def delimiter
+      @delimiter
+    end
+
+    def initialize(path, inode, stat, initial)
+      @path = path
+      @bytes_read = 0
+      @inode = inode
+      @initial = initial
+      @state_history = []
+      @state = :watched
+      @filestat = stat
+      set_accessed_at
+    end
+
+    def init_vars(delim, ignore_o, close_o)
+      @delimiter = delim
+      @ignore_older = ignore_o
+      @close_older = close_o
+      self
+    end
+
+    def set_accessed_at
+      @accessed_at = Time.now.to_f
+    end
+
+    def initial?
+      @initial
+    end
+
+    def size_changed?
+      filestat.size != bytes_read
+    end
+
+    def inode_changed?(value)
+      self.inode != value
+    end
+
+    def file_add_opened(rubyfile)
+      @file = rubyfile
+      @buffer = FileWatch::BufferedTokenizer.new(delimiter || "\n")
+    end
+
+    def file_close
+      return if @file.nil? || @file.closed?
+      @file.close
+      @file = nil
+    end
+
+    def file_seek(amount, whence = IO::SEEK_SET)
+      @file.sysseek(amount, whence)
+    end
+
+    def file_read(amount)
+      set_accessed_at
+      @file.sysread(amount)
+    end
+
+    def file_open?
+      !@file.nil? && !@file.closed?
+    end
+
+    def update_bytes_read(total_bytes_read)
+      return if total_bytes_read.nil?
+      @bytes_read = total_bytes_read
+    end
+
+    def buffer_extract(data)
+      @buffer.extract(data)
+    end
+
+    def update_inode(_inode)
+      @inode = _inode
+    end
+
+    def activate
+      set_state :active
+    end
+
+    def ignore
+      set_state :ignored
+      @bytes_read = @filestat.size
+    end
+
+    def close
+      set_state :closed
+    end
+
+    def watch
+      set_state :watched
+    end
+
+    def unwatch
+      set_state :unwatched
+    end
+
+    def active?
+      @state == :active
+    end
+
+    def ignored?
+      @state == :ignored
+    end
+
+    def closed?
+      @state == :closed
+    end
+
+    def watched?
+      @state == :watched
+    end
+
+    def unwatched?
+      @state == :unwatched
+    end
+
+    def expiry_close_enabled?
+      !@close_older.nil?
+    end
+
+    def expiry_ignore_enabled?
+      !@ignore_older.nil?
+    end
+
+    def restat
+      @filestat = File::Stat.new(path)
+    end
+
+    def set_state(value)
+      @state_history << @state
+      @state = value
+    end
+
+    def state_history_any?(*previous)
+      (@state_history & previous).any?
+    end
+
+    def file_closable?
+      file_can_close? && !size_changed?
+    end
+
+    def file_ignorable?
+      return false unless expiry_ignore_enabled?
+      # (Time.now - stat.mtime) <- in jruby, this does int and float
+      # conversions before the subtraction and returns a float.
+      # so use all floats upfront
+      (Time.now.to_f - filestat.mtime.to_f) > ignore_older
+    end
+
+    def file_can_close?
+      return false unless expiry_close_enabled?
+      (Time.now.to_f - @accessed_at) > close_older
+    end
+
+    def to_s() inspect; end
+  end
+end

data/lib/filewatch/yielding_tail.rb

@@ -8,39 +8,41 @@ module FileWatch
     def subscribe(&block)
       # subscribe(stat_interval = 1, discover_interval = 5, &block)
       @watch.subscribe(@opts[:stat_interval],
-                       @opts[:discover_interval]) do |event, path|
+                       @opts[:discover_interval]) do |event, watched_file|
+        path = watched_file.path
+        file_is_open = watched_file.file_open?
+
         case event
+        when :unignore
+          _add_to_sincedb(watched_file, event)
         when :create, :create_initial
-          if @files.member?(path)
-            @logger.debug? && @logger.debug("#{event} for #{path}: already exists in @files")
+          if file_is_open
+            @logger.debug? && @logger.debug("#{event} for #{path}: file already open")
             next
           end
-          if _open_file(path, event)
-            yield_read_file(path, &block)
+          if _open_file(watched_file, event)
+            yield_read_file(watched_file, &block)
           end
         when :modify
-          if !@files.member?(path)
-            @logger.debug? && @logger.debug(":modify for #{path}, does not exist in @files")
-            if _open_file(path, event)
-              yield_read_file(path, &block)
+          if !file_is_open
+            @logger.debug? && @logger.debug(":modify for #{path}, file is not open, opening now")
+            if _open_file(watched_file, event)
+              yield_read_file(watched_file, &block)
             end
           else
-            yield_read_file(path, &block)
+            yield_read_file(watched_file, &block)
           end
         when :delete
-          @logger.debug? && @logger.debug(":delete for: #{path} - closed and deleted from @files")
-          if @files[path]
-            yield_read_file(path, &block)
-            @files[path].close
+          if file_is_open
+            @logger.debug? && @logger.debug(":delete for #{path}, closing file")
+            yield_read_file(watched_file, &block)
+            watched_file.file_close
+          else
+            @logger.debug? && @logger.debug(":delete for #{path}, file already closed")
           end
-          @files.delete(path)
-          @statcache.delete(path)
         when :timeout
-          @logger.debug? && @logger.debug(":timeout for: #{path} - closed and deleted from @files")
-          if (deleted = @files.delete(path))
-            deleted.close
-          end
-          @statcache.delete(path)
+          @logger.debug? && @logger.debug(":timeout for #{path}, closing file")
+          watched_file.file_close
         else
           @logger.warn("unknown event type #{event} for #{path}")
         end
@@ -48,18 +50,19 @@ module FileWatch
     end # def subscribe
 
     private
-    def yield_read_file(path, &block)
-      @buffers[path] ||= FileWatch::BufferedTokenizer.new(@opts[:delimiter])
-      delimiter_byte_size = @opts[:delimiter].bytesize
+    def yield_read_file(watched_file, &block)
       changed = false
       loop do
         begin
-          data = @files[path].sysread(32768)
+          data = watched_file.file_read(32768)
           changed = true
-          @buffers[path].extract(data).each do |line|
-            yield(path, line)
-            @sincedb[@statcache[path]] += (line.bytesize + delimiter_byte_size)
+          watched_file.buffer_extract(data).each do |line|
+            yield(watched_file.path, line)
+            @sincedb[watched_file.inode] += (line.bytesize + @delimiter_byte_size)
           end
+          # watched_file bytes_read tracks the sincedb entry
+          # see TODO in watch.rb
+          watched_file.update_bytes_read(@sincedb[watched_file.inode])
         rescue Errno::EWOULDBLOCK, Errno::EINTR, EOFError
           break
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filewatch
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Jordan Sissel
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-04 00:00:00.000000000 Z
+date: 2016-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -42,6 +42,7 @@ files:
 - lib/filewatch/tail.rb
 - lib/filewatch/tail_base.rb
 - lib/filewatch/watch.rb
+- lib/filewatch/watched_file.rb
 - lib/filewatch/winhelper.rb
 - lib/filewatch/yielding_tail.rb
 - test/filewatch/tail.rb