eventmachine-tail 0.0.1 → 0.6.1

data/bin/emtail

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "eventmachine"
+require "eventmachine-tail"
+require "optparse"
+
+class Reader < EventMachine::FileTail
+  def initialize(path, startpos=-1, with_filenames=true)
+    super(path, startpos)
+    @buffer = BufferedTokenizer.new
+    @with_filenames = with_filenames
+  end
+
+  def receive_data(data)
+    @buffer.extract(data).each do |line|
+      if @with_filenames # global flag, see the '-n' option
+        puts "#{path}: #{line}"
+      else
+        puts line
+      end # if @with_filenames
+    end # buffer extract
+  end # def receive_data
+end # class Reader
+
+def pattern_to_regexp(pattern)
+  pattern.gsub!(".", "\\.") # fix literal .
+  pattern.gsub!("*", ".+")  # * becomes .+
+  pattern.gsub!("?", ".")   # ? becomes .
+  return Regexp.new(pattern)
+end # def pattern_to_regexp
+
+def main(args)
+  with_filenames = true
+  globcheck_interval = 5
+  exclude_patterns = []
+
+  opts = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options] <path_or_glob> [path_or_glob2] [...]"
+
+    opts.on("-n", "--no-filename", 
+            "Supress prefixing of output with file names") do |x|
+      with_filenames = false
+    end # -n
+    
+    opts.on("-i SECONDS", "--check-interval SECONDS",
+            "How frequently, in seconds, to check the glob patterns" \
+            "for new files") do |x|
+      globcheck_interval = x.to_f
+    end # -i SECONDS
+
+    opts.on("-x EXCLUDE", "--exclude EXCLUDE",
+            "A pattern to ignore. Wildcard/globs accepted." \
+            " Can be specified multiple times") do |pattern|
+      exclude_patterns << pattern_to_regexp(pattern)
+    end
+  end # OptionParser
+
+  opts.parse!(args)
+
+  if args.length == 0
+    puts opts.banner
+    return 1
+  end
+
+  EventMachine.run do
+    Signal.trap("INT") do
+      EventMachine.schedule do
+        $stderr.puts "Got SIGINT"
+        exit 128 + (Signal.list["INT"])
+      end
+    end
+
+    args.each do |path|
+      EventMachine::FileGlobWatchTail.new(path, Reader,
+                                          interval = globcheck_interval,
+                                          exclude = exclude_patterns,
+                                          start_pos = -1,
+                                          with_filenames = with_filenames)
+    end # args.each
+  end # EventMachine.run
+end # def main
+
+exit(main(ARGV))

data/bin/rtail

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "eventmachine"
+require "eventmachine-tail"
+require "optparse"
+
+class Reader < EventMachine::FileTail
+  def initialize(path, startpos=-1, with_filenames=true)
+    super(path, startpos)
+    @buffer = BufferedTokenizer.new
+    @with_filenames = with_filenames
+  end
+
+  def receive_data(data)
+    @buffer.extract(data).each do |line|
+      if @with_filenames # global flag, see the '-n' option
+        puts "#{path}: #{line}"
+      else
+        puts line
+      end # if @with_filenames
+    end # buffer extract
+  end # def receive_data
+end # class Reader
+
+def pattern_to_regexp(pattern)
+  pattern.gsub!(".", "\\.") # fix literal .
+  pattern.gsub!("*", ".+")  # * becomes .+
+  pattern.gsub!("?", ".")   # ? becomes .
+  return Regexp.new(pattern)
+end # def pattern_to_regexp
+
+def main(args)
+  with_filenames = true
+  globcheck_interval = 5
+  exclude_patterns = []
+
+  opts = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options] <path_or_glob> [path_or_glob2] [...]"
+
+    opts.on("-n", "--no-filename", 
+            "Supress prefixing of output with file names") do |x|
+      with_filenames = false
+    end # -n
+    
+    opts.on("-i SECONDS", "--check-interval SECONDS",
+            "How frequently, in seconds, to check the glob patterns" \
+            "for new files") do |x|
+      globcheck_interval = x.to_f
+    end # -i SECONDS
+
+    opts.on("-x EXCLUDE", "--exclude EXCLUDE",
+            "A pattern to ignore. Wildcard/globs accepted." \
+            " Can be specified multiple times") do |pattern|
+      exclude_patterns << pattern_to_regexp(pattern)
+    end
+  end # OptionParser
+
+  opts.parse!(args)
+
+  if args.length == 0
+    puts opts.banner
+    return 1
+  end
+
+  EventMachine.run do
+    Signal.trap("INT") do
+      EventMachine.schedule do
+        $stderr.puts "Got SIGINT"
+        exit 128 + (Signal.list["INT"])
+      end
+    end
+
+    args.each do |path|
+      EventMachine::FileGlobWatchTail.new(path, Reader,
+                                          interval = globcheck_interval,
+                                          exclude = exclude_patterns,
+                                          start_pos = -1,
+                                          with_filenames = with_filenames)
+    end # args.each
+  end # EventMachine.run
+end # def main
+
+exit(main(ARGV))

data/lib/em/filetail.rb

@@ -1,162 +1,447 @@
 #!/usr/bin/env ruby
 
-require "rubygems" if __FILE__ == $0
 require "eventmachine"
 require "logger"
 
 EventMachine.epoll if EventMachine.epoll?
+EventMachine.kqueue = true if EventMachine.kqueue?
 
+# Tail a file.
+#
+# Example
+#   class Tailer < EventMachine::FileTail
+#     def receive_data(data)
+#       puts "Got #{data.length} bytes"
+#     end
+#
+#     # Optional
+#     def eof
+#       puts "Got EOF!"
+#       # If you want to stop
+#       stop
+#     end
+#   end
+#
+#   # Now add it to EM
+#   EM.run do
+#     EM.file_tail("/var/log/messages", Tailer)
+#   end
+#
+#   # Or this way:
+#   EM.run do
+#     Tailer.new("/var/log/messages")
+#   end
+#
+# See also: EventMachine::FileTail#receive_data
 class EventMachine::FileTail
+  # Maximum size to read at a time from a single file.
   CHUNKSIZE = 65536 
-  MAXSLEEP = 2
 
+  #MAXSLEEP = 2
+  
+  FORCE_ENCODING = !! (defined? Encoding)
+
+  # The path of the file being tailed
   attr_reader :path
 
+  # The current file read position
+  attr_reader :position
+
+  # Check interval when checking symlinks for changes. This is only useful
+  # when you are actually tailing symlinks.
+  attr_accessor :symlink_check_interval
+
+  # Check interval for looking for a file if we are tailing it and it has
+  # gone missing.
+  attr_accessor :missing_file_check_interval
+  
+  # Tail a file
+  #
+  # * path is a string file path to tail
+  # * startpos is an offset to start tailing the file at. If -1, start at end of 
+  # file.
+  #
+  # If you want debug messages, run ruby with '-d' or set $DEBUG
+  #
+  # See also: EventMachine::file_tail
+  #
   public
-  def initialize(path, startpos=0)
+  def initialize(path, startpos=-1, &block)
     @path = path
-    @logger = Logger.new(STDOUT)
+    @logger = Logger.new(STDERR)
+    @logger.level = ($DEBUG and Logger::DEBUG or Logger::WARN)
+    @logger.debug("Tailing #{path} starting at position #{startpos}")
 
-    #@need_scheduling = true
-    open
+    @file = nil
+    @want_eof_handling = false
+    @want_read = false
+    @want_reopen = false
+    @reopen_on_eof = false
+    @symlink_timer = nil
+    @symlink_target = nil
+    @symlink_stat = nil
 
-    @fstat = File.stat(@path)
-    @file.seek(0, IO::SEEK_END)
-    watch
+    @symlink_check_interval = 1
+    @missing_file_check_interval = 1
+
+    read_file_metadata
+
+    if @filestat.directory?
+      on_exception Errno::EISDIR.new(@path)
+    end
+
+    if block_given?
+      @handler = block
+      @buffer = BufferedTokenizer.new
+    end
+
+    EventMachine::next_tick do
+      open
+      if (startpos == -1)
+        @position = @file.sysseek(0, IO::SEEK_END)
+        # TODO(sissel): if we don't have inotify or kqueue, should we
+        # schedule a next read, here?
+        # Is there a race condition between setting the file position and
+        # watching given the two together are not atomic?
+      else
+        @position = @file.sysseek(startpos, IO::SEEK_SET)
+        schedule_next_read
+      end
+      watch
+    end # EventMachine::next_tick
   end # def initialize
 
+  # This method is called when a tailed file has data read. 
+  #
+  # * data - string data read from the file.
+  #
+  # If you want to read lines from your file, you should use BufferedTokenizer
+  # (which comes with EventMachine):
+  #   class Tailer < EventMachine::FileTail
+  #     def initialize(*args)
+  #       super(*args)
+  #       @buffer = BufferedTokenizer.new
+  #     end
+  #
+  #     def receive_data(data)
+  #       @buffer.extract(data).each do |line|
+  #         # do something with 'line'
+  #       end
+  #     end  
+  public
+  def receive_data(data)
+    if @handler # FileTail.new called with a block
+      @buffer.extract(data).each do |line|
+        @handler.call(self, line)
+      end
+    else
+      on_exception NotImplementedError.new("#{self.class.name}#receive_data is not "\
+        "implemented. Did you forget to implement this in your subclass or "\
+        "module?")
+    end
+  end # def receive_data
+
+  def on_exception(exception)
+    @logger.error("Exception raised. Using default handler in #{self.class.name}")
+    raise exception
+  end
+
+  # This method is called when a tailed file reaches EOF.
+  #
+  # If you want to stop reading this file, call close(), otherwise
+  # this eof is handled as normal tailing does. The default
+  # EOF handler is to do nothing.
   public
+  def eof
+    @logger.debug { 'EOF' }
+    # do nothing, subclassers should implement this.
+  end # def eof
+
+  # notify is invoked by EM::watch_file when the file you are tailing has been
+  # modified or otherwise needs to be acted on.
+  private
   def notify(status)
-    @logger.debug("#{status} on #{path}")
+    @logger.debug { "notify: #{status} on #{path}" }
     if status == :modified
       schedule_next_read
     elsif status == :moved
-      # TODO(sissel): read to EOF, then reopen.
-      open
+      # read to EOF, then reopen.
+      @reopen_on_eof = true
+      schedule_next_read
+    elsif status == :unbind
+      # :unbind is called after the :deleted handler
+      # :deleted happens on FreeBSD's newsyslog instead of :moved
+      # clean up @watch since its reference is wiped in EM's file_deleted callback
+      @watch = nil
     end
-  end
+  end # def notify
 
+  # Open (or reopen, if necessary) our file and schedule a read.
   private
   def open
+    return if @closed
     @file.close if @file
+    return unless File.exists?(@path)
     begin
+      @logger.debug "Opening file #{@path}"
       @file = File.open(@path, "r")
-    rescue Errno::ENOENT
-      # no file found
-      raise
+    rescue Errno::ENOENT => e
+      @logger.info("File not found: '#{@path}' (#{e})")
+      on_exception(e)
     end
 
-    @naptime = 0;
-    @pos = 0
+    @naptime = 0
+    @logger.debug { 'EOF' }
+    @position = 0
     schedule_next_read
-  end
+  end # def open
 
+  # Close this filetail
+  public
+  def close
+    @closed = true
+    EM.schedule do
+      @file.close if @file
+    end
+  end # def close
+
+  # Watch our file.
   private
   def watch
-    EventMachine::watch_file(@path, FileWatcher, self)
-  end
+    @watch.stop_watching if @watch
+    @symlink_timer.cancel if @symlink_timer
+    return unless File.exists?(@path)
+
+    @logger.debug "Starting watch on #{@path}"
+    callback = proc { |what| notify(what) }
+    @watch = EventMachine::watch_file(@path, EventMachine::FileTail::FileWatcher, callback)
+    watch_symlink if @symlink_target
+  end # def watch
+
+  # Watch a symlink
+  # EM doesn't currently support watching symlinks alone (inotify follows
+  # symlinks by default), so let's periodically stat the symlink.
+  private
+  def watch_symlink(&block)
+    @symlink_timer.cancel if @symlink_timer
+
+    @logger.debug "Launching timer to check for symlink changes since EM can't right now: #{@path}"
+    @symlink_timer = EM::PeriodicTimer.new(@symlink_check_interval) do
+      begin
+        @logger.debug("Checking #{@path}")
+        read_file_metadata do |filestat, linkstat, linktarget|
+          handle_fstat(filestat, linkstat, linktarget)
+        end
+      rescue Errno::ENOENT
+        # The file disappeared. Wait for it to reappear.
+        # This can happen if it was deleted or moved during log rotation.
+        @logger.debug "File not found, waiting for it to reappear. (#{@path})"
+      end # begin/rescue ENOENT
+    end # EM::PeriodicTimer
+  end # def watch_symlink
 
   private
   def schedule_next_read
-    EventMachine::add_timer(@naptime) do
-      read
-    end
-  end
+    if !@want_read
+      @want_read = true
+      EventMachine::add_timer(@naptime) do
+        @want_read = false
+        read
+      end
+    end # if !@want_read
+  end # def schedule_next_read
 
+  # Read CHUNKSIZE from our file and pass it to .receive_data()
   private
   def read
+    return if @closed
+    @logger.debug "#{self}: Reading..."
     begin
       data = @file.sysread(CHUNKSIZE)
+      data.force_encoding(@file.external_encoding) if FORCE_ENCODING
+
       # Won't get here if sysread throws EOF
-      @pos += data.length
+      @position += data.length
       @naptime = 0
+
+      # Subclasses should implement receive_data
       receive_data(data)
       schedule_next_read
     rescue EOFError
-      eof
+      schedule_eof
     end
-  end
+  end # def read
 
+  # Do EOF handling on next EM iteration
   private
-  def eof
-    # TODO(sissel): This will be necessary if we can't use inotify or kqueue to
-    # get notified of file changes
-    #if @need_scheduling
-      #@naptime = 0.100 if @naptime == 0
-      #@naptime *= 2
-      #@naptime = MAXSLEEP if @naptime > MAXSLEEP
-      #@logger.info("EOF. Naptime: #{@naptime}")
-    #end
-
-    # TODO(sissel): schedule an fstat instead of doing it now.
-    fstat = File.stat(@path)
-    handle_fstat(fstat)
-  end # def eof
+  def schedule_eof
+    if !@want_eof_handling
+      eof # Call our own eof event
+      @want_eof_handling = true
+      EventMachine::next_tick do
+        handle_eof
+      end # EventMachine::next_tick
+    end # if !@want_eof_handling
+  end # def schedule_eof
+
+  private
+  def schedule_reopen
+    if !@want_reopen
+      EventMachine::next_tick do
+        @want_reopen = false
+        open
+        watch
+      end
+    end # if !@want_reopen
+  end # def schedule_reopen
+
+  private
+  def handle_eof
+    @want_eof_handling = false
+
+    if @reopen_on_eof
+      @reopen_on_eof = false
+      schedule_reopen
+    end
+
+    # EOF actions:
+    # - Check if the file inode/device is changed
+    # - If symlink, check if the symlink has changed
+    # - Otherwise, do nothing
+    begin
+      read_file_metadata do |filestat, linkstat, linktarget|
+        handle_fstat(filestat, linkstat, linktarget)
+      end
+    rescue Errno::ENOENT
+        # The file disappeared. Wait for it to reappear.
+        # This can happen if it was deleted or moved during log rotation.
+      timer = EM::PeriodicTimer.new(@missing_file_check_interval) do
+        begin
+          read_file_metadata do |filestat, linkstat, linktarget|
+            handle_fstat(filestat, linkstat, linktarget)
+          end
+          timer.cancel
+        rescue Errno::ENOENT
+          # The file disappeared. Wait for it to reappear.
+          # This can happen if it was deleted or moved during log rotation.
+          @logger.debug "File not found, waiting for it to reappear. (#{@path})"
+        end # begin/rescue ENOENT
+      end # EM::PeriodicTimer
+    end # begin/rescue ENOENT
+  end # def handle_eof
+
+  private
+  def read_file_metadata(&block)
+    begin
+      filestat = File.stat(@path)
+    rescue => e
+      @logger.debug("File stat on '#{@path}' failed")
+      on_exception e
+    end
+    symlink_stat = nil
+    symlink_target = nil
+
+    if File.symlink?(@path)
+      symlink_stat = File.lstat(@path) rescue nil
+      symlink_target = File.readlink(@path) rescue nil
+    end
+
+    if block_given?
+      yield filestat, symlink_stat, symlink_target
+    end
 
-  def handle_fstat(fstat)
-    if (fstat.ino != @fstat.ino)
-      open # Reopen if the inode has changed
-    elsif (fstat.rdev != @fstat.rdev)
-      open # Reopen if the filesystem device changed
-    elsif (fstat.size < @fstat.size)
+    @filestat = filestat
+    @symlink_stat = symlink_stat
+    @symlink_target = symlink_target
+  end # def read_file_metadata
+
+  # Handle fstat changes appropriately.
+  private
+  def handle_fstat(filestat, symlinkstat, symlinktarget)
+    # If the symlink target changes, the filestat.ino is very likely to have
+    # changed since that is the stat on the resolved file (that the link points
+    # to). However, we'll check explicitly for the symlink target changing
+    # for better debuggability.
+    if symlinktarget
+      if symlinkstat.ino != @symlink_stat.ino
+        @logger.debug "Inode or device changed on symlink. Reopening..."
+        @reopen_on_eof = true
+        schedule_next_read
+      elsif symlinktarget != @symlink_target
+        @logger.debug "Symlink target changed. Reopening..."
+        @reopen_on_eof = true
+        schedule_next_read
+      end 
+    elsif (filestat.ino != @filestat.ino or filestat.rdev != @filestat.rdev)
+      @logger.debug "Inode or device changed. Reopening..."
+      @logger.debug filestat
+      @reopen_on_eof = true
+      schedule_next_read
+    elsif (filestat.size < @filestat.size)
+      # If the file size shrank, assume truncation and seek to the beginning.
       @logger.info("File likely truncated... #{path}")
-      @file.seek(0, IO::SEEK_SET)
+      @position = @file.sysseek(0, IO::SEEK_SET)
       schedule_next_read
     end
-    @fstat = fstat
-  end # def eof
+  end # def handle_fstat
+
+  def to_s
+    return "#{self.class.name}(#{@path}) @ pos:#{@position}"
+  end # def to_s
 end # class EventMachine::FileTail
 
-# Internal usage only
+# Internal usage only. This class is used by EventMachine::FileTail
+# to watch files you are tailing.
+#
+# See also: EventMachine::FileTail#watch
 class EventMachine::FileTail::FileWatcher < EventMachine::FileWatch
-  def initialize(filestream)
-    @filestream = filestream
-    @logger = Logger.new(STDOUT)
-  end
+  def initialize(block)
+    @logger = Logger.new(STDERR)
+    @logger.level = ($DEBUG and Logger::DEBUG or Logger::WARN)
+    @callback = block
+  end # def initialize
 
   def file_modified
-    @filestream.notify :modified
-  end
+    @callback.call(:modified)
+  end # def file_modified
 
   def file_moved
-    @filestream.notify :moved
-  end
+    @callback.call(:moved)
+  end # def file_moved
 
   def file_deleted
-    @filestream.notify :deleted
-  end
+    @callback.call(:deleted)
+  end # def file_deleted
 
   def unbind
-    @filestream.notify :unbind
-  end
+    @callback.call(:unbind)
+  end # def unbind
 end # class EventMachine::FileTail::FileWatch < EventMachine::FileWatch
 
 # Add EventMachine::file_tail
 module EventMachine
-  def self.file_tail(path, handler=nil, *args)
-    args.unshift(path)
+  # Tail a file.
+  #
+  # path is the path to the file to tail.
+  # handler should be a module implementing 'receive_data' or
+  # must be a subclasses of EventMachine::FileTail
+  #
+  # For example:
+  #   EM::file_tail("/var/log/messages", MyHandler)
+  #
+  # If a block is given, and the handler is not specified or does
+  # not implement EventMachine::FileTail#receive_data, then it
+  # will be called as such:
+  #   EM::file_tail(...) do |filetail, line|
+  #     # filetail is the FileTail instance watching the file
+  #     # line is the line read from the file
+  #   end
+  def self.file_tail(path, handler=nil, *args, &block)
+    # This code mostly styled on what EventMachine does in many of it's other
+    # methods.
+    args = [path, *args]
     klass = klass_from_handler(EventMachine::FileTail, handler, *args);
-    c = klass.new(*args)
-    yield c if block_given?
+    c = klass.new(*args, &block)
     return c
   end # def self.file_tail
 end # module EventMachine
-
-if __FILE__ == $0
-  class Reader < EventMachine::FileTail
-    def initialize(*args)
-      super(*args)
-      @buffer = BufferedTokenizer.new
-    end
-
-    def receive_data(data)
-      @buffer.extract(data).each do |line|
-        ap [path, line]
-      end
-    end
-  end
-
-  EventMachine::run do
-    fw = EventMachine::filestream("/var/log/user.log", Reader)
-  end
-end # if __FILE__ == $0