filewatch 0.6.7 → 0.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c7e0e544aadf722674dd85b5ba0441dce6ecfc4
-  data.tar.gz: 4f6d362df8d9d5f105bcbf12f576e0cf8db69966
+  metadata.gz: d9bf179bb78fa086471f144cc90404a616076d09
+  data.tar.gz: e0579d790be4bc6a400f4db10097cf610fbc219a
 SHA512:
-  metadata.gz: 5fa00a10ea8b093e6484aabbd9f653a7d2f06b97dafdcf2a2f2795f750cf2c3f0d8c13b19f2c41809e5284ed44316e7b448eda1499f7c9b817309a53a34d33c9
-  data.tar.gz: 0c8f460f91f9830deb1b096c22facc1106ea99a73ec297ad41384787bfa29f5302760696411d6b2c3517bb9b3eaa226343a0895ac78eb298ee1a04309b16a51a
+  metadata.gz: 49599f9831b7f87285e819f926f5748f318dcd636313968f00013dc62bf969f43fa74e119374de73bc36a9ef7f39f5ec198f15397892a48e605287fdd216b828
+  data.tar.gz: 80afb0abad3f869aeb7461f7de3d9e9e49b9bbaf010d9c37f4a9803f5cd3d006224aa65d56ac789869a92f0efe974558f16d6f5185566bc686447e09264b8e96

data/lib/filewatch/observing_tail.rb

@@ -0,0 +1,100 @@
+require 'filewatch/tail_base'
+
+module FileWatch
+  class ObservingTail
+    include TailBase
+    public
+
+    class NullListener
+      def initialize(path) @path = path; end
+      def accept(line) end
+      def deleted() end
+      def created() end
+      def error() end
+      def eof() end
+      def timed_out() end
+    end
+
+    class NullObserver
+      def listener_for(path) NullListener.new(path); end
+    end
+
+    def subscribe(observer = NullObserver.new)
+      @watch.subscribe(@opts[:stat_interval],
+                       @opts[:discover_interval]) do |event, path|
+        listener = observer.listener_for(path)
+        case event
+        when :create, :create_initial
+          if @files.member?(path)
+            @logger.debug? && @logger.debug("#{event} for #{path}: already exists in @files")
+            next
+          end
+          if _open_file(path, event)
+            listener.created
+            observe_read_file(path, listener)
+          end
+        when :modify
+          if !@files.member?(path)
+            @logger.debug? && @logger.debug(":modify for #{path}, does not exist in @files")
+            if _open_file(path, event)
+              observe_read_file(path, listener)
+            end
+          else
+            observe_read_file(path, listener)
+          end
+        when :delete
+          @logger.debug? && @logger.debug(":delete for #{path}, deleted from @files")
+          if @files[path]
+            observe_read_file(path, listener)
+            @files[path].close
+          end
+          listener.deleted
+          @files.delete(path)
+          @statcache.delete(path)
+        when :timeout
+          @logger.debug? && @logger.debug(":timeout for #{path}, deleted from @files")
+          if (deleted = @files.delete(path))
+            deleted.close
+          end
+          listener.timed_out
+          @statcache.delete(path)
+        else
+          @logger.warn("unknown event type #{event} for #{path}")
+        end
+      end # @watch.subscribe
+    end # def subscribe
+
+    private
+    def observe_read_file(path, listener)
+      @buffers[path] ||= FileWatch::BufferedTokenizer.new(@opts[:delimiter])
+      delimiter_byte_size = @opts[:delimiter].bytesize
+      changed = false
+      loop do
+        begin
+          data = @files[path].sysread(32768)
+          changed = true
+          @buffers[path].extract(data).each do |line|
+            listener.accept(line)
+            @sincedb[@statcache[path]] += (line.bytesize + delimiter_byte_size)
+          end
+        rescue EOFError
+          listener.eof
+          break
+        rescue Errno::EWOULDBLOCK, Errno::EINTR
+          listener.error
+          break
+        end
+      end
+
+      if changed
+        now = Time.now.to_i
+        delta = now - @sincedb_last_write
+        if delta >= @opts[:sincedb_write_interval]
+          @logger.debug? && @logger.debug("writing sincedb (delta since last write = #{delta})")
+          _sincedb_write
+          @sincedb_last_write = now
+        end
+      end
+    end # def _read_file
+  end
+end # module FileWatch

data/lib/filewatch/tail.rb

@@ -1,272 +1,23 @@
-require "filewatch/helper"
-require "filewatch/buftok"
-require "filewatch/watch"
-if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
-  require "filewatch/winhelper"
-end
-require "logger"
-require "rbconfig"
-
-include Java if defined? JRUBY_VERSION
-require "JRubyFileExtension.jar" if defined? JRUBY_VERSION
+require "filewatch/yielding_tail"
+require "filewatch/observing_tail"
+require "forwardable"
 
 module FileWatch
   class Tail
-    # how often (in seconds) we @logger.warn a failed file open, per path.
-    OPEN_WARN_INTERVAL = ENV["FILEWATCH_OPEN_WARN_INTERVAL"] ?
-                         ENV["FILEWATCH_OPEN_WARN_INTERVAL"].to_i : 300
-
-    attr_accessor :logger
-
-    class NoSinceDBPathGiven < StandardError; end
-
-    public
-    def initialize(opts={})
-      @iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
-
-      if opts[:logger]
-        @logger = opts[:logger]
-      else
-        @logger = Logger.new(STDERR)
-        @logger.level = Logger::INFO
-      end
-      @files = {}
-      @lastwarn = Hash.new { |h, k| h[k] = 0 }
-      @buffers = {}
-      @watch = FileWatch::Watch.new
-      @watch.logger = @logger
-      @sincedb = {}
-      @sincedb_last_write = 0
-      @statcache = {}
-      @opts = {
-        :sincedb_write_interval => 10,
-        :stat_interval => 1,
-        :discover_interval => 5,
-        :exclude => [],
-        :start_new_files_at => :end,
-        :delimiter => "\n"
-      }.merge(opts)
-      if !@opts.include?(:sincedb_path)
-        @opts[:sincedb_path] = File.join(ENV["HOME"], ".sincedb") if ENV.include?("HOME")
-        @opts[:sincedb_path] = ENV["SINCEDB_PATH"] if ENV.include?("SINCEDB_PATH")
-      end
-      if !@opts.include?(:sincedb_path)
-        raise NoSinceDBPathGiven.new("No HOME or SINCEDB_PATH set in environment. I need one of these set so I can keep track of the files I am following.")
-      end
-      @watch.exclude(@opts[:exclude])
-
-      _sincedb_open
-    end # def initialize
-
-    public
-    def logger=(logger)
-      @logger = logger
-      @watch.logger = logger
-    end # def logger=
-
-    public
-    def tail(path)
-      @watch.watch(path)
-    end # def tail
+    extend Forwardable
 
-    public
-    def subscribe(&block)
-      # subscribe(stat_interval = 1, discover_interval = 5, &block)
-      @watch.subscribe(@opts[:stat_interval],
-                       @opts[:discover_interval]) do |event, path|
-        case event
-        when :create, :create_initial
-          if @files.member?(path)
-            @logger.debug? && @logger.debug("#{event} for #{path}: already exists in @files")
-            next
-          end
-          if _open_file(path, event)
-            _read_file(path, &block)
-          end
-        when :modify
-          if !@files.member?(path)
-            @logger.debug? && @logger.debug(":modify for #{path}, does not exist in @files")
-            if _open_file(path, event)
-              _read_file(path, &block)
-            end
-          else
-            _read_file(path, &block)
-          end
-        when :delete
-          @logger.debug? && @logger.debug(":delete for #{path}, deleted from @files")
-          if @files[path]
-            _read_file(path, &block)
-            @files[path].close
-          end
-          @files.delete(path)
-          @statcache.delete(path)
-        else
-          @logger.warn("unknown event type #{event} for #{path}")
-        end
-      end # @watch.subscribe
-    end # def subscribe
+    def_delegators :@target, :tail, :logger=, :subscribe, :sincedb_record_uid, :sincedb_write, :quit, :close_file
 
-    public
-    def sincedb_record_uid(path, stat)
-      inode = @watch.inode(path,stat)
-      @statcache[path] = inode
-      return inode
-    end # def sincedb_record_uid
+    attr_writer :target
 
-    private
-    def _open_file(path, event)
-      @logger.debug? && @logger.debug("_open_file: #{path}: opening")
-      begin
-        if @iswindows && defined? JRUBY_VERSION
-            @files[path] = Java::RubyFileExt::getRubyFile(path)
-        else
-            @files[path] = File.open(path)
-        end
-      rescue
-        # don't emit this message too often. if a file that we can't
-        # read is changing a lot, we'll try to open it more often,
-        # and might be spammy.
-        now = Time.now.to_i
-        if now - @lastwarn[path] > OPEN_WARN_INTERVAL
-          @logger.warn("failed to open #{path}: #{$!}")
-          @lastwarn[path] = now
-        else
-          @logger.debug? && @logger.debug("(warn supressed) failed to open #{path}: #{$!}")
-        end
-        @files.delete(path)
-        return false
+    def self.new_observing(opts = {})
+      new.tap do |instance|
+        instance.target = ObservingTail.new(opts)
       end
-
-      stat = File::Stat.new(path)
-      sincedb_record_uid = sincedb_record_uid(path, stat)
-
-      if @sincedb.member?(sincedb_record_uid)
-        last_size = @sincedb[sincedb_record_uid]
-        @logger.debug? && @logger.debug("#{path}: sincedb last value #{@sincedb[sincedb_record_uid]}, cur size #{stat.size}")
-        if last_size <= stat.size
-          @logger.debug? && @logger.debug("#{path}: sincedb: seeking to #{last_size}")
-          @files[path].sysseek(last_size, IO::SEEK_SET)
-        else
-          @logger.debug? && @logger.debug("#{path}: last value size is greater than current value, starting over")
-          @sincedb[sincedb_record_uid] = 0
-        end
-      elsif event == :create_initial && @files[path]
-        if @opts[:start_new_files_at] == :beginning
-          @logger.debug? && @logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
-          @files[path].sysseek(0, IO::SEEK_SET)
-          @sincedb[sincedb_record_uid] = 0
-        else
-          # seek to end
-          @logger.debug? && @logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
-          @files[path].sysseek(stat.size, IO::SEEK_SET)
-          @sincedb[sincedb_record_uid] = stat.size
-        end
-      elsif event == :create && @files[path]
-        @sincedb[sincedb_record_uid] = 0
-      else
-        @logger.debug? && @logger.debug("#{path}: staying at position 0, no sincedb")
-      end
-
-      return true
-    end # def _open_file
-
-    private
-    def _read_file(path, &block)
-      @buffers[path] ||= FileWatch::BufferedTokenizer.new(@opts[:delimiter])
-      delimiter_byte_size = @opts[:delimiter].bytesize
-      changed = false
-      loop do
-        begin
-          data = @files[path].sysread(32768)
-          changed = true
-          @buffers[path].extract(data).each do |line|
-            yield(path, line)
-            @sincedb[@statcache[path]] += (line.bytesize + delimiter_byte_size)
-          end
-        rescue Errno::EWOULDBLOCK, Errno::EINTR, EOFError
-          break
-        end
-      end
-
-      if changed
-        now = Time.now.to_i
-        delta = now - @sincedb_last_write
-        if delta >= @opts[:sincedb_write_interval]
-          @logger.debug? && @logger.debug("writing sincedb (delta since last write = #{delta})")
-          _sincedb_write
-          @sincedb_last_write = now
-        end
-      end
-    end # def _read_file
-
-    public
-    def sincedb_write(reason=nil)
-      @logger.debug? && @logger.debug("caller requested sincedb write (#{reason})")
-      _sincedb_write
     end
 
-    private
-    def _sincedb_open
-      path = @opts[:sincedb_path]
-      begin
-        db = File.open(path)
-      rescue
-        #No existing sincedb to load
-        @logger.debug? && @logger.debug("_sincedb_open: #{path}: #{$!}")
-        return
-      end
-
-      @logger.debug? && @logger.debug("_sincedb_open: reading from #{path}")
-      db.each do |line|
-        ino, dev_major, dev_minor, pos = line.split(" ", 4)
-        sincedb_record_uid = [ino, dev_major.to_i, dev_minor.to_i]
-        @logger.debug? && @logger.debug("_sincedb_open: setting #{sincedb_record_uid.inspect} to #{pos.to_i}")
-        @sincedb[sincedb_record_uid] = pos.to_i
-      end
-      db.close
-    end # def _sincedb_open
-
-    private
-    def _sincedb_write
-      path = @opts[:sincedb_path]
-      if @iswindows || File.device?(path)
-        IO.write(path, serialize_sincedb, 0)
-      else
-        File.atomic_write(path) {|file| file.write(serialize_sincedb) }
-      end
-    end # def _sincedb_write
-
-    public
-    # quit is a sort-of finalizer,
-    # it should be called for clean up
-    # before the instance is disposed of.
-    def quit
-      _sincedb_write
-      @watch.quit
-      @files.each {|path, file| file.close }
-      @files.clear
-    end # def quit
-
-    public
-    # close_file(path) is to be used by external code
-    # when it knows that it is completely done with a file.
-    # Other files or folders may still be being watched.
-    # Caution, once unwatched, a file can't be watched again
-    # unless a new instance of this class begins watching again.
-    # The sysadmin should rename, move or delete the file.
-    def close_file(path)
-      @watch.unwatch(path)
-      file = @files.delete(path)
-      return if file.nil?
-      _sincedb_write
-      file.close
+    def initialize(opts = {})
+      @target = YieldingTail.new(opts)
     end
-
-    private
-    def serialize_sincedb
-      @sincedb.map do |inode, pos|
-        [inode, pos].flatten.join(" ")
-      end.join("\n") + "\n"
-    end
-  end # class Tail
-end # module FileWatch
+  end
+end

data/lib/filewatch/tail_base.rb

@@ -0,0 +1,220 @@
+require "filewatch/helper"
+require "filewatch/buftok"
+require "filewatch/watch"
+
+if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
+  require "filewatch/winhelper"
+end
+require "logger"
+require "rbconfig"
+
+include Java if defined? JRUBY_VERSION
+require "JRubyFileExtension.jar" if defined? JRUBY_VERSION
+
+module FileWatch
+  module TailBase
+    # how often (in seconds) we @logger.warn a failed file open, per path.
+    OPEN_WARN_INTERVAL = ENV["FILEWATCH_OPEN_WARN_INTERVAL"] ?
+                         ENV["FILEWATCH_OPEN_WARN_INTERVAL"].to_i : 300
+
+    attr_reader :logger
+
+    class NoSinceDBPathGiven < StandardError; end
+
+    public
+    def initialize(opts={})
+      @iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
+
+      if opts[:logger]
+        @logger = opts[:logger]
+      else
+        @logger = Logger.new(STDERR)
+        @logger.level = Logger::INFO
+      end
+      @files = {}
+      @lastwarn = Hash.new { |h, k| h[k] = 0 }
+      @buffers = {}
+      @watch = FileWatch::Watch.new
+      @watch.logger = @logger
+      @sincedb = {}
+      @sincedb_last_write = 0
+      @statcache = {}
+      @opts = {
+        :sincedb_write_interval => 10,
+        :stat_interval => 1,
+        :discover_interval => 5,
+        :exclude => [],
+        :start_new_files_at => :end,
+        # :ignore_after => 24 * 60 * 60,
+        :delimiter => "\n"
+      }.merge(opts)
+      if !@opts.include?(:sincedb_path)
+        @opts[:sincedb_path] = File.join(ENV["HOME"], ".sincedb") if ENV.include?("HOME")
+        @opts[:sincedb_path] = ENV["SINCEDB_PATH"] if ENV.include?("SINCEDB_PATH")
+      end
+      if !@opts.include?(:sincedb_path)
+        raise NoSinceDBPathGiven.new("No HOME or SINCEDB_PATH set in environment. I need one of these set so I can keep track of the files I am following.")
+      end
+      @watch.exclude(@opts[:exclude])
+      @watch.ignore_after = @opts[:ignore_after]
+
+      _sincedb_open
+    end # def initialize
+
+    public
+    def logger=(logger)
+      @logger = logger
+      @watch.logger = logger
+    end # def logger=
+
+    public
+    def tail(path)
+      @watch.watch(path)
+    end # def tail
+
+    public
+    def sincedb_record_uid(path, stat)
+      inode = @watch.inode(path,stat)
+      @statcache[path] = inode
+      return inode
+    end # def sincedb_record_uid
+
+    private
+
+    def file_expired?(stat)
+      return false if @opts[:ignore_after].nil?
+      # (Time.now - stat.mtime) <- in jruby, this does int and float
+      # conversions before the subtraction and returns a float.
+      # so use all ints instead
+      (Time.now.to_i - stat.mtime.to_i) > @opts[:ignore_after]
+    end
+
+    def _open_file(path, event)
+      @logger.debug? && @logger.debug("_open_file: #{path}: opening")
+      begin
+        if @iswindows && defined? JRUBY_VERSION
+          @files[path] = Java::RubyFileExt::getRubyFile(path)
+        else
+          @files[path] = File.open(path)
+        end
+      rescue
+        # don't emit this message too often. if a file that we can't
+        # read is changing a lot, we'll try to open it more often,
+        # and might be spammy.
+        now = Time.now.to_i
+        if now - @lastwarn[path] > OPEN_WARN_INTERVAL
+          @logger.warn("failed to open #{path}: #{$!}")
+          @lastwarn[path] = now
+        else
+          @logger.debug? && @logger.debug("(warn supressed) failed to open #{path}: #{$!}")
+        end
+        @files.delete(path)
+        return false
+      end
+
+      stat = File::Stat.new(path)
+      sincedb_record_uid = sincedb_record_uid(path, stat)
+
+      expired_based_size = file_expired?(stat) ? stat.size : 0
+
+      if @sincedb.member?(sincedb_record_uid)
+        last_size = @sincedb[sincedb_record_uid]
+        @logger.debug? && @logger.debug("#{path}: sincedb last value #{@sincedb[sincedb_record_uid]}, cur size #{stat.size}")
+        if last_size <= stat.size
+          @logger.debug? && @logger.debug("#{path}: sincedb: seeking to #{last_size}")
+          @files[path].sysseek(last_size, IO::SEEK_SET)
+        else
+          @logger.debug? && @logger.debug("#{path}: last value size is greater than current value, starting over")
+          @sincedb[sincedb_record_uid] = 0
+        end
+      elsif event == :create_initial && @files[path]
+        if @opts[:start_new_files_at] == :beginning
+          @logger.debug? && @logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
+          @files[path].sysseek(expired_based_size, IO::SEEK_SET)
+          @sincedb[sincedb_record_uid] = expired_based_size
+        else
+          # seek to end
+          @logger.debug? && @logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
+          @files[path].sysseek(stat.size, IO::SEEK_SET)
+          @sincedb[sincedb_record_uid] = stat.size
+        end
+      elsif event == :create && @files[path]
+        @sincedb[sincedb_record_uid] = expired_based_size
+      else
+        @logger.debug? && @logger.debug("#{path}: staying at position 0, no sincedb")
+      end
+
+      return true
+    end # def _open_file
+
+    public
+    def sincedb_write(reason=nil)
+      @logger.debug? && @logger.debug("caller requested sincedb write (#{reason})")
+      _sincedb_write
+    end
+
+    private
+    def _sincedb_open
+      path = @opts[:sincedb_path]
+      begin
+        db = File.open(path)
+      rescue
+        #No existing sincedb to load
+        @logger.debug? && @logger.debug("_sincedb_open: #{path}: #{$!}")
+        return
+      end
+
+      @logger.debug? && @logger.debug("_sincedb_open: reading from #{path}")
+      db.each do |line|
+        ino, dev_major, dev_minor, pos = line.split(" ", 4)
+        sincedb_record_uid = [ino, dev_major.to_i, dev_minor.to_i]
+        @logger.debug? && @logger.debug("_sincedb_open: setting #{sincedb_record_uid.inspect} to #{pos.to_i}")
+        @sincedb[sincedb_record_uid] = pos.to_i
+      end
+      db.close
+    end # def _sincedb_open
+
+    private
+    def _sincedb_write
+      path = @opts[:sincedb_path]
+      if @iswindows || File.device?(path)
+        IO.write(path, serialize_sincedb, 0)
+      else
+        File.atomic_write(path) {|file| file.write(serialize_sincedb) }
+      end
+    end # def _sincedb_write
+
+    public
+    # quit is a sort-of finalizer,
+    # it should be called for clean up
+    # before the instance is disposed of.
+    def quit
+      _sincedb_write
+      @watch.quit
+      @files.each {|path, file| file.close }
+      @files.clear
+    end # def quit
+
+    public
+    # close_file(path) is to be used by external code
+    # when it knows that it is completely done with a file.
+    # Other files or folders may still be being watched.
+    # Caution, once unwatched, a file can't be watched again
+    # unless a new instance of this class begins watching again.
+    # The sysadmin should rename, move or delete the file.
+    def close_file(path)
+      @watch.unwatch(path)
+      file = @files.delete(path)
+      return if file.nil?
+      _sincedb_write
+      file.close
+    end
+
+    private
+    def serialize_sincedb
+      @sincedb.map do |inode, pos|
+        [inode, pos].flatten.join(" ")
+      end.join("\n") + "\n"
+    end
+  end # module TailBase
+end # module FileWatch