eventmachine-tail 0.2.20100525165012 → 0.3.20100829013522

data/lib/em/filetail.rb

@@ -63,14 +63,16 @@ class EventMachine::FileTail
       raise Errno::EISDIR.new(@path)
     end
 
-    open
-    watch { |what| notify(what) }
-    if (startpos == -1)
-      @file.sysseek(0, IO::SEEK_END)
-    else
-      @file.sysseek(startpos, IO::SEEK_SET)
-      schedule_next_read
-    end
+    EventMachine::next_tick do
+      open
+      watch { |what| notify(what) }
+      if (startpos == -1)
+        @file.sysseek(0, IO::SEEK_END)
+      else
+        @file.sysseek(startpos, IO::SEEK_SET)
+        schedule_next_read
+      end
+    end # EventMachine::next_tick
   end # def initialize
 
   # This method is called when a tailed file has data read. 
@@ -112,7 +114,8 @@ class EventMachine::FileTail
       schedule_next_read
     elsif status == :moved
       # TODO(sissel): read to EOF, then reopen.
-      open
+      # If the file was moved, treat it like EOF.
+      eof
     end
   end
 
@@ -121,6 +124,7 @@ class EventMachine::FileTail
   def open
     @file.close if @file
     begin
+      @logger.debug "Opening file #{@path}"
       @file = File.open(@path, "r")
     rescue Errno::ENOENT
       # no file found
@@ -135,7 +139,8 @@ class EventMachine::FileTail
   # Watch our file.
   private
   def watch(&block)
-    EventMachine::watch_file(@path, EventMachine::FileTail::FileWatcher, block)
+    @logger.debug "Starting watch on #{@path}"
+    @watch = EventMachine::watch_file(@path, EventMachine::FileTail::FileWatcher, block)
   end
 
   # Schedule a read.
@@ -149,6 +154,7 @@ class EventMachine::FileTail
   # Read CHUNKSIZE from our file and pass it to .receive_data()
   private
   def read
+    @logger.debug "#{self}: Reading..."
     begin
       data = @file.sysread(CHUNKSIZE)
       # Won't get here if sysread throws EOF
@@ -173,25 +179,49 @@ class EventMachine::FileTail
     #end
 
     # TODO(sissel): should we schedule an fstat instead of doing it now?
-    fstat = File.stat(@path)
-    handle_fstat(fstat)
+    begin
+      fstat = File.stat(@path)
+      handle_fstat(fstat)
+    rescue Errno::ENOENT
+      # The file disappeared. Wait for it to reappear.
+      # This can happen if it was deleted or moved during log rotation.
+      @logger.debug "File not found, waiting for it to reappear. (#{@path})"
+      timer = EM::PeriodicTimer.new(0.250) do
+        begin
+          fstat = File.stat(@path)
+          handle_fstat(fstat)
+          timer.cancel
+        rescue Errno::ENOENT
+          # ignore
+        end # begin/rescue ENOENT
+      end # EM::PeriodicTimer
+    end # begin/rescue ENOENT
   end # def eof
 
   # Handle fstat changes appropriately.
   private
   def handle_fstat(fstat)
-    if (fstat.ino != @fstat.ino)
-      open # Reopen if the inode has changed
-    elsif (fstat.rdev != @fstat.rdev)
-      open # Reopen if the filesystem device changed
+    if (fstat.ino != @fstat.ino or fstat.rdev != @fstat.rdev)
+      EventMachine::next_tick do
+        @logger.debug "Inode or device changed. Reopening..."
+        @watch.stop_watching
+        open # Reopen if the inode has changed
+        watch { |what| notify(what) }
+      end
     elsif (fstat.size < @fstat.size)
       # Schedule a read if the file size has changed
       @logger.info("File likely truncated... #{path}")
       @file.sysseek(0, IO::SEEK_SET)
       schedule_next_read
+    else
+      #@logger.debug "Nothing to run for fstat..."
     end
     @fstat = fstat
   end # def eof
+
+  def to_s
+    return "#{self.class.name}(#{@path}) @ pos:#{@file.sysseek(0, IO::SEEK_CUR)}"
+  end # def to_s
 end # class EventMachine::FileTail
 
 # Internal usage only. This class is used by EventMachine::FileTail

data/lib/em/globwatcher.rb

@@ -35,7 +35,7 @@ class EventMachine::FileGlobWatch
   # * interval - number of seconds between scanning the glob for changes
   def initialize(glob, interval=60)
     @glob = glob
-    @files = Set.new
+    @files = Hash.new
     @watches = Hash.new
     @logger = Logger.new(STDOUT)
     @logger.level = ($DEBUG and Logger::DEBUG or Logger::WARN)
@@ -83,29 +83,37 @@ class EventMachine::FileGlobWatch
   private
   def find_files
     @logger.info("Searching for files in #{@glob}")
-    list = Set.new(Dir.glob(@glob))
+    list = Dir.glob(@glob)
+
+    known_files = @files.clone
     list.each do |path|
-      next if @files.include?(path)
-      add(path)
+      fileinfo = FileInfo.new(path) rescue next
+      # Skip files that have the same inode (renamed or hardlinked)
+      known_files.delete(fileinfo.stat.ino)
+      next if @files.include?(fileinfo.stat.ino)
+
+      track(fileinfo)
+      file_found(path)
     end
 
-    (@files - list).each do |missing|
-      remove(missing)
+    # Report missing files.
+    known_files.each do |inode, fileinfo|
+      remove(fileinfo)
     end
   end # def find_files
 
   # Remove a file from being watched and notify file_deleted()
   private
-  def remove(path)
-    @files.delete(path)
-    @watches.delete(path)
-    file_deleted(path)
+  def remove(fileinfo)
+    @files.delete(fileinfo.stat.ino)
+    @watches.delete(fileinfo.path)
+    file_deleted(fileinfo.path)
   end # def remove
 
   # Add a file to watch and notify file_found()
   private
-  def add(path)
-    @files.add(path)
+  def track(fileinfo)
+    @files[fileinfo.stat.ino] = fileinfo
 
     # If EventMachine::watch_file fails, that's ok, I guess.
     # We'll still find the file 'missing' from the next glob attempt.
@@ -120,7 +128,6 @@ class EventMachine::FileGlobWatch
     #rescue Errno::EACCES => e
       #@logger.warn(e)
     #end
-    file_found(path)
   end # def watch
 
   private
@@ -139,6 +146,17 @@ class EventMachine::FileGlobWatch
       block.call path
     end
   end # class EventMachine::FileGlobWatch::FileWatcher < EventMachine::FileWatch
+
+  private 
+  class FileInfo
+    attr_reader :path
+    attr_reader :stat
+
+    def initialize(path)
+      @path = path
+      @stat = File.stat(path)
+    end
+  end # class FileInfo
 end # class EventMachine::FileGlobWatch
 
 # A glob tailer for EventMachine

data/test/test_filetail.rb

@@ -92,5 +92,53 @@ class TestFileTail < Test::Unit::TestCase
       end
     end # EM.run
   end # def test_filetail_with_block
+
+  def test_filetail_tracks_renames
+    tmp = Tempfile.new("testfiletail")
+    data = DATA.clone
+    filename = tmp.path
+
+    data_copy = data.clone
+
+    # Write first so the first read happens immediately
+    tmp.puts data_copy.shift
+    tmp.flush
+    EM.run do
+      abort_after_timeout(DATA.length * SLEEPMAX + 10)
+
+      lineno = 0
+      # Start at file position 0.
+      EM::file_tail(tmp.path, nil, 0) do |filetail, line|
+        lineno += 1
+        expected = data.shift
+        #puts "Got #{lineno}: #{line}"
+        assert_equal(expected, line, 
+                     "Expected '#{expected}' on line #{lineno}, but got '#{line}'")
+        finish if data.length == 0
+
+        # Start a timer on the first read.
+        # This is to ensure we have the file tailing before
+        # we try to rename.
+        if lineno == 1
+          timer = EM::PeriodicTimer.new(0.2) do
+            value = data_copy.shift
+            tmp.puts value
+            tmp.flush
+            sleep(rand * SLEEPMAX)
+
+            # Rename the file, create a new one in it's place.
+            # This is to simulate log rotation, etc.
+            path_newname = "#{filename}_#{value}"
+            File.rename(filename, path_newname)
+            File.delete(path_newname)
+            tmp = File.open(filename, "w")
+            timer.cancel if data_copy.length == 0
+          end # timer
+        end # if lineno == 1
+      end # EM::filetail(...)
+    end # EM.run
+
+    File.delete(filename)
+  end # def test_filetail_tracks_renames
 end # class TestFileTail
 

data/test/test_glob.rb

@@ -86,11 +86,61 @@ class TestGlobWatcher < Test::Unit::TestCase
       datacopy = @data.clone
       timer = EM::PeriodicTimer.new(0.2) do
         #puts "Creating: #{datacopy.first}"
-        File.new(datacopy.shift, "w")
+        File.new(datacopy.shift, "w").close
         sleep(rand * SLEEPMAX)
         timer.cancel if datacopy.length == 0
       end
     end # EM.run
   end # def test_glob_finds_newly_created_files_at_runtime
+
+  def test_glob_ignores_file_renames
+    EM.run do
+      abort_after_timeout(SLEEPMAX * @data.length + 10)
+
+      EM::watch_glob("#{@dir}/*", Watcher, @watchinterval, @data.clone, self)
+
+      datacopy = @data.clone
+      timer = EM::PeriodicTimer.new(0.2) do
+        filename = datacopy.shift
+        File.new(filename, "w").close
+        sleep(rand * SLEEPMAX)
+
+        # This file rename should be ignored.
+        EM::Timer.new(2) do
+          newname = "#{filename}.renamed"
+          File.rename(filename, newname)
+
+          # Track the new filename so teardown removes it.
+          @data << newname
+        end
+        timer.cancel if datacopy.length == 0
+      end
+    end
+  end # def test_glob_ignores_file_renames
+
+  def test_glob_ignores_duplicate_hardlinks
+    EM.run do
+      abort_after_timeout(SLEEPMAX * @data.length + 10)
+
+      EM::watch_glob("#{@dir}/*", Watcher, @watchinterval, @data.clone, self)
+
+      datacopy = @data.clone
+      timer = EM::PeriodicTimer.new(0.2) do
+        filename = datacopy.shift
+        File.new(filename, "w").close
+        sleep(rand * SLEEPMAX)
+
+        # This file rename should be ignored.
+        EM::Timer.new(2) do
+          newname = "#{filename}.renamed"
+          File.link(filename, newname)
+
+          # Track the new filename so teardown removes it.
+          @data << newname
+        end
+        timer.cancel if datacopy.length == 0
+      end
+    end
+  end # def test_glob_ignores_file_renames
 end # class TestGlobWatcher
 

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 2
-  - 20100525165012
-  version: 0.2.20100525165012
+  - 3
+  - 20100829013522
+  version: 0.3.20100829013522
 platform: ruby
 authors: 
 - Jordan Sissel
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-25 00:00:00 -07:00
+date: 2010-08-29 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency