filewatch 0.2.5 → 0.3.0

data/bin/globtail

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require "filewatch/tail"
+require "optparse"
+
+progname = File.basename($0)
+
+config = {
+  :verbose => false,
+}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "#{progname} [-v] [-s path] [-i interval] [-x glob] path/glob ..."
+
+  opts.on("-v", "--verbose", "Enable verbose/debug output") do
+    config[:verbose] = true
+  end
+
+  opts.on("-x", "--exclude PATH", String, "path to exclude from watching") do |path|
+    config[:exclude] ||= []
+    config[:exclude] << path
+  end
+
+  opts.on("-s", "--sincedb PATH", String, "Sincedb path") do |path|
+    config[:sincedb_path] = path
+  end
+
+  opts.on("-i", "--interval SECONDS", Integer,
+          "Sincedb write interval") do |path|
+    config[:sincedb_write_interval] = path
+  end
+end
+
+begin
+  opts.order!
+rescue OptionParser::InvalidOption
+  $stderr.puts "#{progname}: #{$!}"
+  $stderr.puts opts.usage
+end
+
+logger = Logger.new(STDERR)
+logger.progname = progname
+logger.level = config[:verbose] ? Logger::DEBUG : Logger::INFO
+config[:logger] = logger
+
+tail = FileWatch::Tail.new(config)
+
+ARGV.each { |path| tail.tail(path) }
+
+Signal.trap("EXIT") { tail.sincedb_write("globtail exiting") }
+
+$stdout.sync = true
+tail.subscribe do |path, line|
+  puts "#{path}: #{line}"
+end

data/lib/filewatch/buftok.rb

@@ -29,7 +29,7 @@
 #    end
 #  end
 
-class BufferedTokenizer
+module FileWatch; class BufferedTokenizer
   # New BufferedTokenizers will operate on lines delimited by "\n" by default
   # or allow you to specify any delimiter token you so choose, which will then
   # be used by String#split to tokenize the input data
@@ -136,4 +136,4 @@ cter token support.
   def empty?
     @input.empty?
   end
-end
+end; end

data/lib/filewatch/tail.rb

@@ -1,58 +1,181 @@
+require "filewatch/buftok"
 require "filewatch/watch"
-require "filewatch/namespace"
-
-class FileWatch::Tail
-  # This class exists to wrap inotify, kqueue, periodic polling, etc,
-  # to provide you with a way to watch files and directories.
-  #
-  # For now, it only supports inotify.
-  def initialize
-    @watch = FileWatch::Watch.new
-    @files = {}
-  end
-
-  public
-  def watch(path, *what_to_watch)
-    @watch.watch(path, *what_to_watch)
-
-    if File.file?(path)
-      @files[path] = File.new(path, "r")
-      
-      # TODO(sissel): Support 'since'-like support.
-      # Always start at the end of the file, this may change in the future.
-      @files[path].sysseek(0, IO::SEEK_END)
-    end
-  end # def watch
-
-  def subscribe(handler=nil, &block)
-    @watch.subscribe(nil) do |event|
-      path = event.name
-      if @files.include?(path)
-        file = @files[path]
-        event.actions.each do |action|
-          # call method 'file_action_<action>' like 'file_action_modify'
-          method = "file_action_#{action}".to_sym
-          if respond_to?(method)
-            send(method, file, event, &block)
-          else
-            $stderr.puts "Unsupported method #{self.class.name}##{method}"
+require "logger"
+
+module FileWatch
+  class Tail
+    attr_accessor :logger
+
+    public
+    def initialize(opts={})
+      if opts[:logger]
+        @logger = opts[:logger]
+      else
+        @logger = Logger.new(STDERR)
+        @logger.level = Logger::INFO
+      end
+      @files = {}
+      @buffers = {}
+      @watch = FileWatch::Watch.new
+      @watch.logger = @logger
+      @sincedb = {}
+      @sincedb_last_write = 0
+      @statcache = {}
+      @opts = {
+        :sincedb_write_interval => 10,
+        :sincedb_path => "#{ENV["HOME"]}/.sincedb",
+        :exclude => [],
+      }.merge(opts)
+      @watch.exclude(@opts[:exclude])
+
+      _sincedb_open
+    end # def initialize
+
+    public
+    def logger=(logger)
+      @logger = logger
+      @watch.logger = logger
+    end # def logger=
+
+    public
+    def tail(path)
+      @watch.watch(path)
+    end # def tail
+
+    public
+    def subscribe(&block)
+      # subscribe(stat_interval = 1, discover_interval = 5, &block)
+      @watch.subscribe do |event, path|
+        case event
+        when :create, :create_initial
+          if @files.member?(path)
+            @logger.debug("#{event} for #{path}: already exists in @files")
+            next
           end
+          _open_file(path, event)
+          _read_file(path, &block)
+        when :modify
+          if !@files.member?(path)
+            @logger.debug(":modify for #{path}, does not exist in @files")
+            _open_file(path)
+          end
+          _read_file(path, &block)
+        when :delete
+          @logger.debug(":delete for #{path}, deleted from @files")
+          _read_file(path, &block)
+          @files[path].close
+          @files.delete(path)
+          @statcache.delete(path)
+        else
+          @logger.warn("unknown event type #{event} for #{path}")
+        end
+      end # @watch.subscribe
+    end # def each
+
+    private
+    def _open_file(path, event)
+      @logger.debug("_open_file: #{path}: opening")
+      # TODO(petef): handle File.open failing
+      begin
+        @files[path] = File.open(path)
+      rescue Errno::ENOENT
+        @logger.warn("#{path}: open: #{$!}")
+        @files.delete(path)
+        return
+      end
+
+      stat = File::Stat.new(path)
+      inode = [stat.ino, stat.dev_major, stat.dev_minor]
+      @statcache[path] = inode
+
+      if @sincedb.member?(inode)
+        last_size = @sincedb[inode]
+        @logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{stat.size}")
+        if last_size <= stat.size
+          @logger.debug("#{path}: sincedb: seeking to #{last_size}")
+          @files[path].sysseek(last_size, IO::SEEK_SET)
+        else
+          @logger.debug("#{path}: last value size is greater than current value, starting over")
+          @sincedb[inode] = 0
         end
+      elsif event == :create_initial && @files[path]
+        @logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
+        @files[path].sysseek(stat.size, IO::SEEK_SET)
+        @sincedb[inode] = stat.size
       else
-        $stderr.puts "Event on unwatched file: #{event}"
+        @logger.debug("#{path}: staying at position 0, no sincedb")
       end
+    end # def _open_file
+
+    private
+    def _read_file(path, &block)
+      @buffers[path] ||= FileWatch::BufferedTokenizer.new
+
+      changed = false
+      loop do
+        begin
+          data = @files[path].read_nonblock(4096)
+          changed = true
+          @buffers[path].extract(data).each do |line|
+            yield(path, line)
+          end
+
+          @sincedb[@statcache[path]] = @files[path].pos
+        rescue Errno::EWOULDBLOCK, Errno::EINTR, EOFError
+          break
+        end
+      end
+
+      if changed
+        now = Time.now.to_i
+        delta = now - @sincedb_last_write
+        if delta >= @opts[:sincedb_write_interval]
+          @logger.debug("writing sincedb (delta since last write = #{delta})")
+          _sincedb_write
+          @sincedb_last_write = now
+        end
+      end
+    end # def _read_file
+
+    public
+    def sincedb_write(reason=nil)
+      @logger.debug("caller requested sincedb write (#{reason})")
+      _sincedb_write
     end
-  end # def subscribe
 
-  def file_action_modify(file, event)
-    loop do
+    private
+    def _sincedb_open
+      path = @opts[:sincedb_path]
       begin
-        data = file.sysread(4096)
-        yield event.name, data
-      rescue EOFError
-        break
+        db = File.open(path)
+      rescue
+        @logger.debug("_sincedb_open: #{path}: #{$!}")
+        return
       end
-    end
-  end
 
-end # class FileWatch::Tail
+      @logger.debug("_sincedb_open: reading from #{path}")
+      db.each do |line|
+        ino, dev_major, dev_minor, pos = line.split(" ", 4)
+        inode = [ino.to_i, dev_major.to_i, dev_minor.to_i]
+        @logger.debug("_sincedb_open: setting #{inode.inspect} to #{pos.to_i}")
+        @sincedb[inode] = pos.to_i
+      end
+    end # def _sincedb_open
+
+    private
+    def _sincedb_write
+      path = @opts[:sincedb_path]
+      begin
+        db = File.open(path, "w")
+      rescue
+        @logger.debug("_sincedb_write: #{path}: #{$!}")
+        return
+      end
+
+      @sincedb.each do |inode, pos|
+        db.puts([inode, pos].flatten.join(" "))
+      end
+      db.close
+    end # def _sincedb_write
+  end # class Watch
+end # module FileWatch

data/lib/filewatch/watch.rb

@@ -1,26 +1,145 @@
-require "filewatch/namespace"
-require "filewatch/inotify/fd"
-require "filewatch/exception"
-
-class FileWatch::Watch
-  # This class exists to wrap inotify, kqueue, periodic polling, etc,
-  # to provide you with a way to watch files and directories.
-  #
-  # For now, it only supports inotify.
-  def initialize
-    @inotify = FileWatch::Inotify::FD.new
-  end
-
-  public
-  def watch(path, *what_to_watch)
-    return @inotify.watch(path, *what_to_watch)
-  end # def watch
-
-  def subscribe(handler=nil, &block)
-    @inotify.subscribe(handler, &block)
-  end
-
-  def each(&block)
-    @inotify.each(&block)
-  end # def each
-end # class FileWatch::Watch
+require "logger"
+
+module FileWatch
+  class Watch
+    attr_accessor :logger
+
+    public
+    def initialize(opts={})
+      if opts[:logger]
+        @logger = opts[:logger]
+      else
+        @logger = Logger.new(STDERR)
+        @logger.level = Logger::INFO
+      end
+      @watching = []
+      @exclude = []
+      @files = Hash.new { |h, k| h[k] = Hash.new }
+    end # def initialize
+
+    public
+    def logger=(logger)
+      @logger = logger
+    end
+
+    public
+    def exclude(path)
+      path.to_a.each { |p| @exclude << p }
+    end
+
+    public
+    def watch(path)
+      if ! @watching.member?(path)
+        @watching << path
+        _discover_file(path, true)
+      end
+
+      return true
+    end # def tail
+
+    # Calls &block with params [event_type, path]
+    # event_type can be one of:
+    #   :create_initial - initially present file (so start at end for tail)
+    #   :create - file is created (new file after initial globs, start at 0)
+    #   :modify - file is modified (size increases)
+    #   :delete - file is deleted
+    public
+    def each(&block)
+      # Send any creates.
+      @files.keys.each do |path|
+        if ! @files[path][:create_sent]
+          if @files[path][:initial]
+            yield(:create_initial, path)
+          else
+            yield(:create, path)
+          end
+          @files[path][:create_sent] = true
+        end
+      end
+
+      @files.keys.each do |path|
+        begin
+          stat = File::Stat.new(path)
+        rescue Errno::ENOENT
+          # file has gone away or we can't read it anymore.
+          @files.delete(path)
+          @logger.debug("#{path}: stat failed (#{$!}), deleting from @files")
+          yield(:delete, path)
+          next
+        end
+
+        inode = [stat.ino, stat.dev_major, stat.dev_minor]
+        if inode != @files[path][:inode]
+          @logger.debug("#{path}: old inode was #{@files[path][:inode].inspect}, new is #{inode.inspect}")
+          yield(:delete, path)
+          yield(:create, path)
+        elsif stat.size < @files[path][:size]
+          @logger.debug("#{path}: file rolled, new size is #{stat.size}, old size #{@files[path][:size]}")
+          yield(:delete, path)
+          yield(:create, path)
+        elsif stat.size > @files[path][:size]
+          @logger.debug("#{path}: file grew, old size #{@files[path][:size]}, new size #{stat.size}")
+          yield(:modify, path)
+        end
+
+        @files[path][:size] = stat.size
+        @files[path][:inode] = inode
+      end # @files.keys.each
+    end # def each
+
+    public
+    def discover
+      @watching.each do |path|
+        _discover_file(path)
+      end
+    end
+
+    public
+    def subscribe(stat_interval = 1, discover_interval = 5, &block)
+      glob = 0
+      loop do
+        each(&block)
+
+        glob += 1
+        if glob == discover_interval
+          discover
+          glob = 0
+        end
+
+        sleep(stat_interval)
+      end
+    end # def subscribe
+
+    private
+    def _discover_file(path, initial=false)
+      Dir.glob(path).each do |file|
+        next if @files.member?(file)
+        next unless File.file?(file)
+
+        @logger.debug("_discover_file: #{path}: new: #{file} (exclude is #{@exclude.inspect})")
+
+        skip = false
+        @exclude.each do |pattern|
+          if File.fnmatch?(pattern, File.basename(file))
+            @logger.debug("_discover_file: #{file}: skipping because it " +
+                          "matches exclude #{pattern}")
+            skip = true
+            break
+          end
+        end
+        next if skip
+
+        stat = File::Stat.new(file)
+        @files[file] = {
+          :size => 0,
+          :inode => [stat.ino, stat.dev_major, stat.dev_minor],
+          :create_sent => false,
+        }
+        if initial
+          @files[file][:initial] = true
+        end
+      end
+    end # def _discover_file
+
+  end # class Watch
+end # module FileWatch