eventmachine-tail 0.0.1 → 0.6.1

data/lib/em/globwatcher.rb

@@ -1,87 +1,280 @@
 #!/usr/bin/env ruby
 
-require "rubygems" if __FILE__ == $0
-require "set"
+require "em/filetail"
 require "eventmachine"
-require "ap"
 require "logger"
+require "set"
 
-require "filetail"
+EventMachine.epoll if EventMachine.epoll?
+EventMachine.kqueue = true if EventMachine.kqueue?
 
+# A file glob pattern watcher for EventMachine.
+#
+# If you are unfamiliar with globs, see Wikipedia:
+# http://en.wikipedia.org/wiki/Glob_(programming)
+#
+# Any glob supported by Dir#glob will work with
+# this class.
+#
+# This class will allow you to get notified whenever a file
+# is created or deleted that matches your glob.
+#
+# If you are subclassing, here are the methods you should implement:
+#    file_found(path)
+#    file_deleted(path)
+#
+# See alsoe
+# * EventMachine::watch_glob
+# * EventMachine::FileGlobWatch#file_found
+# * EventMachine::FileGlobWatch#file_deleted
+#
 class EventMachine::FileGlobWatch
-  def initialize(pathglob, handler, interval=60)
-    @pathglob = pathglob
-    @handler = handler
-    @files = Set.new
+  # Watch a glob
+  #
+  # * glob - a string path or glob, such as "/var/log/*.log"
+  # * interval - number of seconds between scanning the glob for changes
+  def initialize(glob, interval=60)
+    @glob = glob
+    @files = Hash.new
+    @watches = Hash.new
+    @logger = Logger.new(STDOUT)
+    @logger.level = ($DEBUG and Logger::DEBUG or Logger::WARN)
 
+    # We periodically check here because it is easier than writing our own glob
+    # parser (so we can smartly watch globs like /foo/*/bar*/*.log)
+    #
+    # Reasons to fix this -
+    # This will likely perform badly on globs that result in a large number of
+    # files.
     EM.next_tick do
       find_files
       EM.add_periodic_timer(interval) do
         find_files
       end
-    end
+    end # EM.next_tick
   end # def initialize
 
+  # This method is called when a new file is found
+  #
+  # * path - the string path of the file found
+  #
+  # You must implement this in your subclass or module for it
+  # to work with EventMachine::watch_glob
+  public
+  def file_found(path)
+    raise NotImplementedError.new("#{self.class.name}#file_found is not "\
+      "implemented. Did you forget to implement this in your subclass or "\
+      "module?")
+  end # def file_found
+
+  # This method is called when a file is deleted.
+  #
+  # * path - the string path of the file deleted
+  #
+  # You must implement this in your subclass or module for it
+  # to work with EventMachine::watch_glob
+  public
+  def file_deleted(path)
+    raise NotImplementedError.new("#{self.class.name}#file_deleted is not "\
+      "implemented. Did you forget to implement this in your subclass or "\
+      "module?")
+  end # def file_found
+
+  private
   def find_files
-    list = Set.new(Dir.glob(@pathglob))
+    @logger.info("Searching for files in #{@glob}")
+    list = Dir.glob(@glob)
+
+    known_files = @files.clone
     list.each do |path|
-      next if @files.include?(path)
-      watch(path)
+      fileinfo = FileInfo.new(path) rescue next
+      # Skip files that have the same inode (renamed or hardlinked)
+      known_files.delete(fileinfo.stat.ino)
+      next if @files.include?(fileinfo.stat.ino)
+
+      track(fileinfo)
+      file_found(path)
     end
 
-    (@files - list).each do |missing|
-      @files.delete(missing)
-      @handler.file_removed(missing)
+    # Report missing files.
+    known_files.each do |inode, fileinfo|
+      remove(fileinfo)
     end
   end # def find_files
 
-  def watch(path)
-    puts "Watching #{path}"
-    @files.add(path)
-    @handler.file_found(path)
+  # Remove a file from being watched and notify file_deleted()
+  private
+  def remove(fileinfo)
+    @files.delete(fileinfo.stat.ino)
+    @watches.delete(fileinfo.path)
+    file_deleted(fileinfo.path)
+  end # def remove
+
+  # Add a file to watch and notify file_found()
+  private
+  def track(fileinfo)
+    @files[fileinfo.stat.ino] = fileinfo
+
+    # If EventMachine::watch_file fails, that's ok, I guess.
+    # We'll still find the file 'missing' from the next glob attempt.
+    #begin
+      # EM currently has a bug that only the first handler for a watch_file
+      # on each file gets events. This causes globtails to never get data 
+      # since the glob is watching the file already.
+      # Until we fix that, let's skip file watching here.
+      #@watches[path] = EventMachine::watch_file(path, FileWatcher, self) do |path|
+      #  remove(path)
+      #end
+    #rescue Errno::EACCES => e
+      #@logger.warn(e)
+    #end
   end # def watch
+
+  private
+  class FileWatcher < EventMachine::FileWatch
+    def initialize(globwatch, &block)
+      @globwatch = globwatch
+      @block = block
+    end
+
+    def file_moved
+      stop_watching
+      block.call path
+    end
+
+    def file_deleted
+      block.call path
+    end
+  end # class EventMachine::FileGlobWatch::FileWatcher < EventMachine::FileWatch
+
+  private 
+  class FileInfo
+    attr_reader :path
+    attr_reader :stat
+
+    def initialize(path)
+      @path = path
+      @stat = File.stat(path)
+    end
+  end # class FileInfo
 end # class EventMachine::FileGlobWatch
 
-class EventMachine::FileGlobWatchHandler
-  LOGGER = Logger.new(STDOUT)
-  def initialize(handler=nil)
+# A glob tailer for EventMachine
+#
+# This class combines features of EventMachine::file_tail and 
+# EventMachine::watch_glob.
+#
+# You won't generally subclass this class (See EventMachine::FileGlobWatch)
+#
+# See also: EventMachine::glob_tail
+#
+class EventMachine::FileGlobWatchTail < EventMachine::FileGlobWatch
+  # Initialize a new file glob tail.
+  #
+  # * path - glob or file path (string)
+  # * handler - a module or subclass of EventMachine::FileTail
+  #   See also EventMachine::file_tail
+  # * interval - how often (seconds) the glob path should be scanned
+  # * exclude - an array of Regexp (or anything with .match) for 
+  #   excluding from things to tail
+  #
+  # The remainder of arguments are passed to EventMachine::file_tail as
+  #   EventMachine::file_tail(path_found, handler, *args, &block)
+  public
+  def initialize(path, handler=nil, interval=60, exclude=[], *args, &block)
+    super(path, interval)
     @handler = handler
-  end
+    @args = args
+    @exclude = exclude
+
+    if block_given?
+      @handler = block
+    end
+  end # def initialize
 
+  public
   def file_found(path)
-    EventMachine::file_tail(path, @handler)
-  end
+    begin
+      @logger.info "#{self.class}: Trying #{path}"
+      @exclude.each do |exclude|
+        @logger.info "#{self.class}: Testing #{exclude} =~ #{path} == #{exclude.match(path) != nil}"
+        if exclude.match(path) != nil
+          file_excluded(path) 
+          return
+        end
+      end
+      @logger.info "#{self.class}: Watching #{path}"
+
+      if @handler.is_a? Proc
+        EventMachine::file_tail(path, nil, *@args, &@handler)
+      else
+        EventMachine::file_tail(path, @handler, *@args)
+      end
+    rescue Errno::EACCES => e
+      file_error(path, e)
+    rescue Errno::EISDIR => e
+      file_error(path, e)
+    end 
+  end # def file_found
+
+  public
+  def file_excluded(path)
+    @logger.info "#{self.class}: Skipping path #{path} due to exclude rule"
+  end # def file_excluded
 
-  def file_removed(path)
+  public
+  def file_deleted(path)
     # Nothing to do
-  end
+  end # def file_deleted
+
+  public
+  def file_error(path, e)
+    $stderr.puts "#{e.class} while trying to tail #{path}"
+    # otherwise, drop the error by default
+  end # def file_error
 end # class EventMachine::FileGlobWatchHandler
 
 module EventMachine
-  def self.glob_tail(glob, handler=nil, *args)
-    handler = EventMachine::FileGlobHandler if handler == nil
-    klass = klass_from_handler(EventMachine::FileGlobWatchHandler, handler, *args)
+  # Watch a glob and tail any files found.
+  #
+  # * glob - a string path or glob, such as /var/log/*.log
+  # * handler - a module or subclass of EventMachine::FileGlobWatchTail.
+  #   handler can be omitted if you give a block.
+  #
+  # If you give a block and omit the handler parameter, then the behavior
+  # is that your block is called for every line read from any file the same
+  # way EventMachine::file_tail does when called with a block.
+  #           
+  #   See EventMachine::FileGlobWatchTail for the callback methods.
+  #   See EventMachine::file_tail for more information about block behavior.
+  def self.glob_tail(glob, handler=nil, *args, &block)
+    handler = EventMachine::FileGlobWatchTail if handler == nil
+    args.unshift(glob)
+    klass = klass_from_handler(EventMachine::FileGlobWatchTail, handler, *args)
+    c = klass.new(*args, &block)
+    return c
+  end 
+
+  # Watch a glob for any files.
+  #
+  # * glob - a string path or glob, such as "/var/log/*.log"
+  # * handler - must be a module or a subclass of EventMachine::FileGlobWatch
+  # 
+  # The remaining (optional) arguments are passed to your handler like this:
+  #   If you call this:
+  #     EventMachine.watch_glob("/var/log/*.log", YourHandler, 1, 2, 3, ...)
+  #   This will be invoked when new matching files are found:
+  #     YourHandler.new(path_found, 1, 2, 3, ...)
+  #     ^ path_found is the new path found by the glob
+  #
+  # See EventMachine::FileGlobWatch for the callback methods.
+  def self.watch_glob(glob, handler=nil, *args)
+    # This code mostly styled on what EventMachine does in many of it's other
+    # methods.
+    args = [glob, *args]
+    klass = klass_from_handler(EventMachine::FileGlobWatch, handler, *args);
     c = klass.new(*args)
     yield c if block_given?
     return c
-  end
-end
-
-if __FILE__ == $0
-  class Reader < EventMachine::FileTail
-    def initialize(*args)
-      super(*args)
-      @buffer = BufferedTokenizer.new
-    end
-
-    def receive_data(data)
-      @buffer.extract(data).each do |line|
-        ap [path, line]
-      end
-    end
-  end
-
-  EventMachine.run do
-    EventMachine::FileGlobWatch.new("/var/log/*.log", EventMachine::FileGlobWatchHandler.new(Reader))
-  end
-end
+  end # def EventMachine::watch_glob
+end # module EventMachine

data/lib/eventmachine-tail.rb

@@ -0,0 +1,2 @@
+require 'em/filetail'
+require 'em/globwatcher'

data/samples/glob-tail.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+#
+# Sample that uses eventmachine-tail to watch a file or set of files.
+# Basically, this example implements 'tail -f' but can accept globs
+# that are also watched.
+#
+# For example, '/var/log/*.log' will be periodically watched for new
+# matching files which will additionally be watched.
+#
+# Usage example:
+#   glob-tail.rb "/var/log/*.log" "/var/log/httpd/*.log"
+#
+# (Important to use quotes or otherwise escape the '*' chars, otherwise
+#  your shell will interpret them)
+
+require "rubygems"
+require "eventmachine"
+require "eventmachine-tail"
+
+class Reader < EventMachine::FileTail
+  def initialize(path, startpos=-1)
+    super(path, startpos)
+    puts "Tailing #{path}"
+    @buffer = BufferedTokenizer.new
+  end
+
+  def receive_data(data)
+    @buffer.extract(data).each do |line|
+      puts "#{path}: #{line}"
+    end
+  end
+end
+
+def main(args)
+  if args.length == 0
+    puts "Usage: #{$0} <path_or_glob> [path_or_glob2] [...]"
+    return 1
+  end
+
+  EventMachine.run do
+    args.each do |path|
+      EventMachine::FileGlobWatchTail.new(path, Reader)
+    end
+  end
+end # def main
+
+exit(main(ARGV))

data/samples/globwatch.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "eventmachine"
+require "eventmachine-tail"
+
+class Watcher < EventMachine::FileGlobWatch
+  def initialize(pathglob, interval=5)
+    super(pathglob, interval)
+  end
+
+  def file_deleted(path)
+    puts "Removed: #{path}"
+  end
+
+  def file_found(path)
+    puts "Found: #{path}"
+  end
+end # class Watcher
+
+EM.run do
+  Watcher.new("/var/log/*")
+end

data/samples/tail-with-block.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+#
+# Simple 'tail -f' example. This one uses a block instead of a separate handler
+# class
+# Usage example:
+#   tail-with-block.rb /var/log/messages
+
+require "rubygems"
+require "eventmachine"
+require "eventmachine-tail"
+
+def main(args)
+  if args.length == 0
+    puts "Usage: #{$0} <path> [path2] [...]"
+    return 1
+  end
+
+  EventMachine.run do
+    args.each do |path|
+      EventMachine::file_tail(path) do |filetail, line|
+        # filetail is the 'EventMachine::FileTail' instance for this file.
+        # line is the line read from thefile.
+        # this block is invoked for every line read.
+        puts line
+      end
+    end
+  end
+end # def main
+
+exit(main(ARGV))

data/samples/tail.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+#
+# Simple 'tail -f' example.
+# Usage example:
+#   tail.rb /var/log/messages
+
+require "rubygems"
+require "eventmachine"
+require "eventmachine-tail"
+
+class Reader < EventMachine::FileTail
+  def initialize(path, startpos=-1)
+    super(path, startpos)
+    puts "Tailing #{path}"
+    @buffer = BufferedTokenizer.new
+  end
+
+  def receive_data(data)
+    @buffer.extract(data).each do |line|
+      puts "#{path}: #{line}"
+    end
+  end
+end
+
+def main(args)
+  if args.length == 0
+    puts "Usage: #{$0} <path> [path2] [...]"
+    return 1
+  end
+
+  EventMachine.run do
+    args.each do |path|
+      EventMachine::file_tail(path, Reader)
+    end
+  end
+end # def main
+
+exit(main(ARGV))

data/test/alltests.rb

@@ -0,0 +1,5 @@
+$: << File.expand_path('..', __FILE__)
+
+
+require 'test_filetail'
+require 'test_glob'

data/test/logrotate.conf

@@ -0,0 +1,4 @@
+/tmp/foo/testfile {
+  rotate 5
+  size 50
+}