filewatch 0.1.2 → 0.2.0

data/bin/gtail

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "filewatch/tailglob"
+require "filewatch/buftok"
+require "optparse"
+
+def main(args)
+  with_filenames = true
+  exclude_patterns = []
+
+  opts = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options] <path_or_glob> [path_or_glob2] [...]"
+
+    opts.on("-n", "--no-filename", 
+            "Supress prefixing of output with file names") do |x|
+      with_filenames = false
+    end # -n
+
+    opts.on("-x EXCLUDE", "--exclude EXCLUDE",
+            "A pattern to ignore. Wildcard/globs accepted." \
+            " Can be specified multiple times") do |pattern|
+      exclude_patterns << pattern
+    end
+  end # OptionParser
+
+  opts.parse!(args)
+
+  if args.length == 0
+    puts opts.banner
+    return 1
+  end
+
+  tail = FileWatch::TailGlob.new
+  ARGV.each do |path|
+    tail.tail(path, :exclude => exclude_patterns)
+  end
+
+  buffer = BufferedTokenizer.new
+  tail.subscribe do |path, data|
+    buffer.extract(data).each do |line|
+      if with_filenames
+        puts "#{path}: #{line}"
+      else
+        puts line
+      end
+    end # buffer.extract
+  end # tail.subscribe
+end # def main
+
+exit(main(ARGV))

data/lib/filewatch/buftok.rb

@@ -0,0 +1,139 @@
+# BufferedTokenizer - Statefully split input data by a specifiable token
+#
+# Authors:: Tony Arcieri, Martin Emde
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Tony Arcieri and Martin Emde
+# 
+# Distributed under the Ruby license (http://www.ruby-lang.org/en/LICENSE.txt)
+#
+#---------------------------------------------------------------------------
+#
+
+# (C)2006 Tony Arcieri, Martin Emde
+# Distributed under the Ruby license (http://www.ruby-lang.org/en/LICENSE.txt)
+
+# BufferedTokenizer takes a delimiter upon instantiation, or acts line-based
+# by default.  It allows input to be spoon-fed from some outside source which
+# receives arbitrary length datagrams which may-or-may-not contain the token
+# by which entities are delimited.
+#
+# Commonly used to parse lines out of incoming data:
+#
+#  module LineBufferedConnection
+#    def receive_data(data)
+#      (@buffer ||= BufferedTokenizer.new).extract(data).each do |line|
+#        receive_line(line)
+#      end
+#    end
+#  end
+
+class BufferedTokenizer
+  # New BufferedTokenizers will operate on lines delimited by "\n" by default
+  # or allow you to specify any delimiter token you so choose, which will then
+  # be used by String#split to tokenize the input data
+  def initialize(delimiter = "\n", size_limit = nil)
+    # Store the specified delimiter
+    @delimiter = delimiter
+
+    # Store the specified size limitation
+    @size_limit = size_limit
+
+    # The input buffer is stored as an array.  This is by far the most efficient
+    # approach given language constraints (in C a linked list would be a more
+    # appropriate data structure).  Segments of input data are stored in a list
+    # which is only joined when a token is reached, substantially reducing the
+    # number of objects required for the operation.
+    @input = []
+
+    # Size of the input buffer
+    @input_size = 0
+  end
+
+  # Extract takes an arbitrary string of input data and returns an array of
+  # tokenized entities, provided there were any available to extract.  This
+  # makes for easy processing of datagrams using a pattern like:
+  #
+  #   tokenizer.extract(data).map { |entity| Decode(entity) }.each do ...
+  def extract(data)
+    # Extract token-delimited entities from the input string with the split command.
+    # There's a bit of craftiness here with the -1 parameter.  Normally split would
+    # behave no differently regardless of if the token lies at the very end of the 
+    # input buffer or not (i.e. a literal edge case)  Specifying -1 forces split to
+    # return "" in this case, meaning that the last entry in the list represents a
+    # new segment of data where the token has not been encountered
+    entities = data.split @delimiter, -1
+
+    # Check to see if the buffer has exceeded capacity, if we're imposing a limit
+    if @size_limit
+      raise 'input buffer full' if @input_size + entities.first.size > @size_limit
+      @input_size += entities.first.size
+    end
+    
+    # Move the first entry in the resulting array into the input buffer.  It represents
+    # the last segment of a token-delimited entity unless it's the only entry in the list.
+    @input << entities.shift
+
+    # If the resulting array from the split is empty, the token was not encountered
+    # (not even at the end of the buffer).  Since we've encountered no token-delimited
+    # entities this go-around, return an empty array.
+    return [] if entities.empty?
+
+    # At this point, we've hit a token, or potentially multiple tokens.  Now we can bring
+    # together all the data we've buffered from earlier calls without hitting a token,
+    # and add it to our list of discovered entities.
+    entities.unshift @input.join
+
+=begin
+    # Note added by FC, 10Jul07. This paragraph contains a regression. It breaks
+    # empty tokens. Think of the empty line that delimits an HTTP header. It will have
+    # two "\n" delimiters in a row, and this code mishandles the resulting empty token.
+    # It someone figures out how to fix the problem, we can re-enable this code branch.
+    # Multi-chara
+cter token support.
+    # Split any tokens that were incomplete on the last iteration buf complete now.
+    entities.map! do |e|
+      e.split @delimiter, -1
+    end
+    # Flatten the resulting array.  This has the side effect of removing the empty
+    # entry at the end that was produced by passing -1 to split.  Add it again if
+    # necessary.
+    if (entities[-1] == [])
+      entities.flatten! << []
+    else
+      entities.flatten!
+    end
+=end
+
+    # Now that we've hit a token, joined the input buffer and added it to the entities
+    # list, we can go ahead and clear the input buffer.  All of the segments that were
+    # stored before the join can now be garbage collected.
+    @input.clear
+    
+    # The last entity in the list is not token delimited, however, thanks to the -1
+    # passed to split.  It represents the beginning of a new list of as-yet-untokenized  
+    # data, so we add it to the start of the list.
+    @input << entities.pop
+    
+    # Set the new input buffer size, provided we're keeping track
+    @input_size = @input.first.size if @size_limit
+
+    # Now we're left with the list of extracted token-delimited entities we wanted
+    # in the first place.  Hooray!
+    entities
+  end
+  
+  # Flush the contents of the input buffer, i.e. return the input buffer even though
+  # a token has not yet been encountered
+  def flush
+    buffer = @input.join
+    @input.clear
+    buffer
+  end
+
+  # Is the buffer empty?
+  def empty?
+    @input.empty?
+  end
+end

data/lib/filewatch/exception.rb

@@ -0,0 +1,12 @@
+require "filewatch/namespace"
+
+class FileWatch::Exception < Exception
+  attr_accessor :fd
+  attr_accessor :path
+
+  def initialize(message, fd, path)
+    super(message)
+    @fd = fd
+    @path = path
+  end
+end

data/lib/filewatch/inotify/event.rb

@@ -12,6 +12,9 @@ class FileWatch::Inotify::Event < FFI::Struct
 
   attr_accessor :name
 
+  # Enum of :directory or :file
+  attr_accessor :type
+
   def initialize(pointer)
     if pointer.is_a?(String)
       pointer = FFI::MemoryPointer.from_string(pointer)
@@ -54,7 +57,11 @@ class FileWatch::Inotify::Event < FFI::Struct
 
   def from_stringpipeio(io)
     begin
-      @name = io.read(self[:len], true)
+      if self[:len] > 0
+        @name = io.read(self[:len], true)
+      else
+        @name = nil
+      end
     rescue Errno::EINVAL => e
       $stderr.puts "Read was too small? Confused."
       raise e
@@ -73,7 +80,7 @@ class FileWatch::Inotify::Event < FFI::Struct
   end
 
   def to_s
-    return "#{@name} (#{self.actions.join(", ")})"
+    return "#{@name} (#{self.actions.join(", ")}) [type=#{type}]"
   end
 
   def partial?

data/lib/filewatch/inotify/fd.rb

@@ -1,8 +1,10 @@
 require "rubygems"
 require "ffi"
+require "fcntl"
 require "filewatch/inotify/event"
 require "filewatch/namespace"
 require "filewatch/stringpipeio"
+require "filewatch/exception"
 
 class FileWatch::Inotify::FD
   include Enumerable
@@ -12,7 +14,7 @@ class FileWatch::Inotify::FD
     ffi_lib FFI::Library::LIBC
 
     attach_function :inotify_init, [], :int
-    attach_function :inotify_init1, [:int], :int
+    attach_function :fcntl, [:int, :int, :long], :int
     attach_function :inotify_add_watch, [:int, :string, :uint32], :int
 
     # So we can read and poll inotify from jruby.
@@ -24,6 +26,9 @@ class FileWatch::Inotify::FD
 
   INOTIFY_CLOEXEC = 02000000
   INOTIFY_NONBLOCK = 04000
+  
+  F_SETFL = 4
+  O_NONBLOCK = 04000  
 
   WATCH_BITS = {
     :access => 1 << 0,
@@ -60,12 +65,19 @@ class FileWatch::Inotify::FD
     @watches = {}
     @buffer = FileWatch::StringPipeIO.new
 
-    @fd = CInotify.inotify_init1(INOTIFY_NONBLOCK)
+    #@fd = CInotify.inotify_init1(INOTIFY_NONBLOCK)
+    @fd = CInotify.inotify_init()
 
     if java?
       @io = nil
+      @rc = CInotify.fcntl(@fd, F_SETFL, O_NONBLOCK)
+      if @rc == -1
+        raise FileWatch::Exception.new(
+          "fcntl(#{@fd}, F_SETFL, O_NONBLOCK) failed. #{$?}", @fd, nil)
+      end
     else
       @io = IO.for_fd(@fd)
+      @io.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
     end
   end
 
@@ -110,10 +122,10 @@ class FileWatch::Inotify::FD
   def watch(path, *what_to_watch)
     mask = what_to_watch.inject(0) { |m, val| m |= WATCH_BITS[val] }
     watch_descriptor = CInotify.inotify_add_watch(@fd, path, mask)
-    #puts "watch #{path} => #{watch_descriptor}"
 
     if watch_descriptor == -1
-      raise "inotify_add_watch(#{@fd}, #{path}, #{mask}) failed. #{$?}"
+      raise FileWatch::Exception.new(
+        "inotify_add_watch(#{@fd}, #{path}, #{mask}) failed. #{$?}", @fd, path)
     end
     @watches[watch_descriptor] = {
       :path => path,
@@ -167,11 +179,13 @@ class FileWatch::Inotify::FD
     if event.name == nil
       # Some events don't have the name at all, so add our own.
       event.name = watchpath
+      event.type = :file
     else
       # Event paths are relative to the watch, if a directory. Prefix to make
       # the full path.
       if watch[:is_directory]
         event.name = File.join(watchpath, event.name)
+        event.type = :directory
       end
     end
 

data/lib/filewatch/tail.rb

@@ -30,6 +30,7 @@ class FileWatch::Tail
       if @files.include?(path)
         file = @files[path]
         event.actions.each do |action|
+          # call method 'file_action_<action>' like 'file_action_modify'
           method = "file_action_#{action}".to_sym
           if respond_to?(method)
             send(method, file, event, &block)

data/lib/filewatch/tailglob.rb

@@ -0,0 +1,182 @@
+require "filewatch/namespace"
+require "filewatch/exception"
+require "filewatch/watchglob"
+
+class FileWatch::TailGlob
+
+  public
+  def initialize
+    @watch = FileWatch::WatchGlob.new
+
+    # hash of string path => File
+    @files = {}
+    @options = {}
+
+    # hash of string path => action to take on EOF
+    #@eof_actions = {}
+  end # def initialize
+
+  # Watch a path glob.
+  #
+  # Options is a hash of:
+  #   :exclude => array of globs to ignore.
+  public
+  def tail(path, options)
+    what_to_watch = [ :create, :modify, :delete ]
+
+    # Save glob options
+    @options[path] = options
+
+    @watch.watch(path, *what_to_watch) do |path|
+      # Save per-path options
+      @options[path] = options
+     
+      # for each file found by the glob, open it.
+      follow_file(path, :end)
+    end # @watch.watch
+  end # def watch
+
+  private
+  def follow_file(path, seek=:end)
+    # Don't follow things that aren't files.
+    if !File.file?(path) 
+      puts "Skipping follow on #{path}, File.file? == false"
+      return
+    end
+
+    options = @options[path] || {}
+    if options.include?(:exclude)
+      options[:exclude].each do |exclusion|
+        if File.fnmatch?(exclusion, path)
+          puts "Skipping #{path}, matches #{exclusion}"
+          return
+        end
+      end
+    end
+
+    close(path) if @files.include?(path)
+    @files[path] = File.new(path, "r")
+    
+    # TODO(sissel): Support 'since'-like support.
+    case seek
+      when :end; @files[path].sysseek(0, IO::SEEK_END)
+      when :beginning; # nothing
+      else
+        if seek.is_a?(Numeric)
+          # 'seek' could be a number that is an offset from
+          # the start of the file. We should seek to that point.
+          @files[path].sysseek(seek, IO::SEEK_SET)
+        end # seek.is_a?(Numeric)
+    end # case seek
+  end # def follow_file
+
+  public
+  def subscribe(handler=nil, &block)
+    # TODO(sissel): Add handler support.
+    @watch.subscribe do |event|
+      path = event.name
+
+      event.actions.each do |action|
+        method = "file_action_#{action}".to_sym
+        if respond_to?(method)
+          send(method, path, event, &block)
+        else
+          $stderr.puts "Unsupported method #{self.class.name}##{method}"
+        end
+      end
+    end # @watch.subscribe
+  end # def subscribe
+
+  protected
+  def file_action_modify(path, event, &block)
+    loop do
+      file = @files[path]
+      begin
+        data = file.sysread(4096)
+        yield event.name, data
+      rescue EOFError
+        check_for_truncation_or_deletion(path, event, &block)
+        #case @eof_actions[path]
+          #when :reopen
+            #puts "Reopening #{path} due to eof and new file"
+            #reopen(path)
+        #end
+
+        break
+      end
+    end
+  end
+
+  protected
+  def check_for_truncation_or_deletion(path, event, &block)
+    file = @files[path]
+    pos = file.sysseek(0, IO::SEEK_CUR)
+    #puts "EOF(#{path}), pos: #{pos}"
+
+    # Truncation is determined by comparing the current read position in the
+    # file against the size of the file. If the file shrank, than we should
+    # assume truncation and seek to the beginning.
+    begin
+      stat = file.stat
+      #p stat.size => pos
+      if stat.size < pos
+        # Truncated. Seek to beginning and read.
+        file.sysseek(0, IO::SEEK_SET)
+        file_action_modify(path, event, &block)
+      end
+    rescue Errno::ENOENT
+      # File was deleted or renamed. Stop following it.
+      close(path)
+    end
+  end # def follow_file
+
+  protected
+  def reopen(path)
+    close(path)
+    follow_file(path, :beginning)
+  end # def reopen
+
+  protected
+  def close(path)
+    @files[path].close rescue nil
+    @files.delete(path)
+    return nil
+  end
+
+  protected
+  def file_action_create(path, event, &block)
+    if following?(path)
+      # TODO(sissel): If we are watching this file already, schedule it to be
+      # opened the next time we hit EOF on the current file descriptor for the
+      # same file. Maybe? Or is reopening now fine?
+      #
+      reopen(path)
+
+      # Schedule a reopen at EOF
+      #@eof_actions[path] = :reopen
+    else
+      # If we are not yet watching this file, watch it.
+      follow_file(path, :beginning)
+
+      # Then read all of the data so far since this is a new file.
+      file_action_modify(path, event, &block)
+    end
+  end # def file_action_create
+
+  def file_action_delete(path, event, &block)
+    close(path)
+    # ignore
+  end
+
+  def file_action_delete_self(path, event, &block)
+    close(path)
+    # ignore
+    #p :delete_self => path
+  end
+
+  # Returns true if we are currently following the file at the given path.
+  public
+  def following?(path)
+    return @files.include?(path)
+  end # def following
+end # class FileWatch::Tail

data/lib/filewatch/watch.rb

@@ -1,5 +1,6 @@
-require "filewatch/inotify/fd"
 require "filewatch/namespace"
+require "filewatch/inotify/fd"
+require "filewatch/exception"
 
 class FileWatch::Watch
   # This class exists to wrap inotify, kqueue, periodic polling, etc,

data/lib/filewatch/watchglob.rb

@@ -0,0 +1,79 @@
+require "filewatch/namespace"
+require "filewatch/exception"
+require "filewatch/watch"
+
+class FileWatch::WatchGlob
+  # This class exists to wrap inotify, kqueue, periodic polling, etc,
+  # to provide you with a way to watch files and directories.
+  #
+  # For now, it only supports inotify.
+  def initialize
+    @watch = FileWatch::Watch.new
+    @globdirs = []
+    @globs = []
+  end
+
+  public
+  def watch(glob, *what_to_watch, &block)
+    @globs << [glob, what_to_watch]
+
+    watching = [] 
+    errors = []
+    paths = Dir.glob(glob)
+    paths.each do |path|
+      begin
+        next if watching.include?(path)
+        puts "Watching #{path}"
+        @watch.watch(path, :create, :delete, :modify)
+        watching << path
+
+        # Yield each file found by glob to the block.
+        # This allows initialization on new files found at start.
+        yield path if block_given?
+      rescue FileWatch::Exception => e
+        $stderr.puts "Failed starting watch on #{path} - #{e}"
+        errors << e
+      end
+    end
+
+    # Go through the glob and look for paths leading into a '*'
+    splitpath = glob.split(File::SEPARATOR)
+    splitpath.each_with_index do |part, i|
+      current = File.join(splitpath[0 .. i])
+      current = "/" if current.empty?
+      next if watching.include?(current)
+      # TODO(sissel): Do better glob detection
+      if part.include?("*")
+        globprefix = File.join(splitpath[0 ... i])
+        Dir.glob(globprefix).each do |path|
+          next if watching.include?(path)
+          p "Watching dir #{path}"
+          @watch.watch(path, :create)
+          @globdirs << path
+        end
+      end
+    end
+  end # def watch
+
+  # TODO(sissel): implement 'unwatch' or cancel?
+
+  def subscribe(handler=nil, &block)
+    @watch.subscribe do |event|
+      # If this event is a directory event and the file matches a watched glob,
+      # then it should be a new-file creation event. Watch the file.
+      if event.type == :directory and @globdirs.include?(File.dirname(event.name))
+        glob, what = @globs.find { |glob, what| File.fnmatch?(glob, event.name) }
+        if glob
+          @watch.watch(event.name, *what)
+        end
+      end
+
+      # Push the event to our callback.
+      block.call event
+    end
+  end
+
+  def each(&block)
+    @inotify.each(&block)
+  end # def each
+end # class FileWatch::Watch

data/test/log4j/LogTest.java

@@ -0,0 +1,21 @@
+import org.apache.log4j.Logger;
+import org.apache.log4j.PropertyConfigurator;
+
+public class LogTest {
+  private static Logger logger = Logger.getLogger(LogTest.class);
+
+  public static void main(String[] args) {
+    //BasicConfigurator.configure();
+    PropertyConfigurator.configure("log4j.properties");
+
+    int i = 0;
+    while (true) {
+      logger.info("Testing: " + i);
+      i++;
+      try {
+        Thread.sleep(100);
+      }  catch (Exception e) {
+      }
+    }
+  } /* public static void main(String[]) */
+} /* public class LogTest */

data/test/log4j/log4j.properties

@@ -0,0 +1,20 @@
+log4j.rootLogger=INFO,stdout,rolling
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%5p %d{HH:mm:ss,SSS} %m%n
+
+log4j.appender.rolling=org.apache.log4j.RollingFileAppender
+log4j.appender.rolling.maxFileSize=1000
+log4j.appender.rolling.maxBackupIndex=10
+log4j.appender.rolling.layout=org.apache.log4j.PatternLayout
+log4j.appender.rolling.layout.ConversionPattern=%5p [%t] %d{ISO8601} %F (line %L) %m%n
+log4j.appender.rolling.File=logs/logtest.log
+log4j.appender.rolling.DatePattern=.yyyy-MM-dd.HH-mm-ss
+
+#log4j.appender.D=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.D.layout=org.apache.log4j.PatternLayout
+#log4j.appender.D.layout=org.apache.log4j.JSONLayout
+#log4j.appender.D.layout.ConversionPattern=%5p [%t] %d{ISO8601} %F (line %L) %m%n
+#log4j.appender.D.File=logs/logtest.log
+#log4j.appender.D.DatePattern=.yyyy-MM-dd.HH-mm-ss
+

metadata

@@ -1,16 +1,21 @@
 --- !ruby/object:Gem::Specification 
 name: filewatch
 version: !ruby/object:Gem::Version 
+  hash: 23
   prerelease: 
-  version: 0.1.2
+  segments: 
+  - 0
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
-  - Jordan Sissel
+- Jordan Sissel
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2011-03-19 00:00:00 -07:00
+date: 2011-04-12 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -23,13 +28,20 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-  - lib/filewatch/watch.rb
-  - lib/filewatch/namespace.rb
-  - lib/filewatch/tail.rb
-  - lib/filewatch/stringpipeio.rb
-  - lib/filewatch/inotify/emhandler.rb
-  - lib/filewatch/inotify/fd.rb
-  - lib/filewatch/inotify/event.rb
+- lib/filewatch/buftok.rb
+- lib/filewatch/exception.rb
+- lib/filewatch/tail.rb
+- lib/filewatch/stringpipeio.rb
+- lib/filewatch/inotify/emhandler.rb
+- lib/filewatch/inotify/fd.rb
+- lib/filewatch/inotify/event.rb
+- lib/filewatch/watchglob.rb
+- lib/filewatch/watch.rb
+- lib/filewatch/namespace.rb
+- lib/filewatch/tailglob.rb
+- test/log4j/log4j.properties
+- test/log4j/LogTest.java
+- bin/gtail
 has_rdoc: true
 homepage: https://github.com/jordansissel/ruby-filewatch
 licenses: []
@@ -38,24 +50,30 @@ post_install_message:
 rdoc_options: []
 
 require_paths: 
-  - lib
-  - lib
+- lib
+- lib
 required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.5.1
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: filewatch - file watching for ruby