filewatch 0.6.7 → 0.6.8

data/lib/filewatch/watch.rb

@@ -5,7 +5,47 @@ end
 
 module FileWatch
   class Watch
-    attr_accessor :logger
+    class WatchedFile
+      def self.new_initial(path, inode)
+        new(path, inode, true)
+      end
+
+      def self.new_ongoing(path, inode)
+        new(path, inode, false)
+      end
+
+      attr_reader :size, :inode
+      attr_writer :create_sent, :initial, :timeout_sent
+
+      attr_reader :path
+
+      def initialize(path, inode, initial)
+        @path = path
+        @size, @create_sent, @timeout_sent = 0, false, false
+        @inode, @initial = inode, initial
+      end
+
+      def update(stat, inode = nil)
+        @size = stat.size
+        @inode = inode if inode
+      end
+
+      def create_sent?
+        @create_sent
+      end
+
+      def initial?
+        @initial
+      end
+
+      def timeout_sent?
+        @timeout_sent
+      end
+
+      def to_s() inspect; end
+    end
+
+    attr_accessor :logger, :ignore_after
 
     public
     def initialize(opts={})
@@ -18,20 +58,18 @@ module FileWatch
       end
       @watching = []
       @exclude = []
-      @files = Hash.new { |h, k| h[k] = Hash.new }
+      @files = Hash.new { |h, k| h[k] = WatchedFile.new(k, false, false) }
       @unwatched = Hash.new
       # we need to be threadsafe about the mutation
       # of the above 2 ivars because the public
       # methods each, discover, watch and unwatch
       # can be called from different threads.
       @lock = Mutex.new
+      # we need to be threadsafe about the quit mutation
+      @quit = false
+      @quit_lock = Mutex.new
     end # def initialize
 
-    public
-    def logger=(logger)
-      @logger = logger
-    end
-
     public
     def exclude(path)
       path.to_a.each { |p| @exclude << p }
@@ -42,7 +80,9 @@ module FileWatch
       synchronized do
         if !@watching.member?(path)
           @watching << path
-          _discover_file(path, true)
+          _discover_file(path) do |filepath, stat|
+            WatchedFile.new_initial(filepath, inode(filepath, stat))
+          end
         end
       end
       return true
@@ -86,18 +126,18 @@ module FileWatch
     def each(&block)
       synchronized do
         # Send any creates.
-        @files.keys.each do |path|
-          if ! @files[path][:create_sent]
-            if @files[path][:initial]
+        @files.each do |path, watched_file|
+          if !watched_file.create_sent?
+            if watched_file.initial?
               yield(:create_initial, path)
             else
               yield(:create, path)
             end
-            @files[path][:create_sent] = true
+            watched_file.create_sent = true
           end
         end
 
-        @files.keys.each do |path|
+        @files.each do |path, watched_file|
           begin
             stat = File::Stat.new(path)
           rescue Errno::ENOENT
@@ -108,23 +148,33 @@ module FileWatch
             next
           end
 
+          if expired?(stat, watched_file)
+            if !watched_file.timeout_sent?
+              @logger.debug? && @logger.debug("#{path}: file expired")
+              yield(:timeout, path)
+              watched_file.timeout_sent = true
+            end
+            next
+          end
+
           inode = inode(path,stat)
-          if inode != @files[path][:inode]
-            @logger.debug? && @logger.debug("#{path}: old inode was #{@files[path][:inode].inspect}, new is #{inode.inspect}")
+          old_size = watched_file.size
+
+          if inode != watched_file.inode
+            @logger.debug? && @logger.debug("#{path}: old inode was #{watched_file.inode.inspect}, new is #{inode.inspect}")
             yield(:delete, path)
             yield(:create, path)
-          elsif stat.size < @files[path][:size]
-            @logger.debug? && @logger.debug("#{path}: file rolled, new size is #{stat.size}, old size #{@files[path][:size]}")
+          elsif stat.size < old_size
+            @logger.debug? && @logger.debug("#{path}: file rolled, new size is #{stat.size}, old size #{old_size}")
             yield(:delete, path)
             yield(:create, path)
-          elsif stat.size > @files[path][:size]
-            @logger.debug? && @logger.debug("#{path}: file grew, old size #{@files[path][:size]}, new size #{stat.size}")
+          elsif stat.size > old_size
+            @logger.debug? && @logger.debug("#{path}: file grew, old size #{old_size}, new size #{stat.size}")
             yield(:modify, path)
           end
 
-          @files[path][:size] = stat.size
-          @files[path][:inode] = inode
-        end # @files.keys.each
+          watched_file.update(stat, inode)
+        end
       end
     end # def each
 
@@ -132,7 +182,9 @@ module FileWatch
     def discover
       synchronized do
         @watching.each do |path|
-          _discover_file(path)
+          _discover_file(path) do |filepath, stat|
+            WatchedFile.new_ongoing(filepath, inode(filepath, stat))
+          end
         end
       end
     end
@@ -140,8 +192,8 @@ module FileWatch
     public
     def subscribe(stat_interval = 1, discover_interval = 5, &block)
       glob = 0
-      @quit = false
-      while !@quit
+      reset_quit
+      while !quit?
         each(&block)
 
         glob += 1
@@ -155,7 +207,24 @@ module FileWatch
     end # def subscribe
 
     private
-    def _discover_file(path, initial=false)
+    def expired?(stat, watched_file)
+      file_expired?(stat) && watched_file.size == stat.size
+    end
+
+    def discover_expired?(stat)
+      file_expired?(stat)
+    end
+
+    def file_expired?(stat)
+      return false unless expiry_enabled?
+      # (Time.now - stat.mtime) <- in jruby, this does int and float
+      # conversions before the subtraction and returns a float.
+      # so use all ints instead
+      (Time.now.to_i - stat.mtime.to_i) > @ignore_after
+    end
+
+    private
+    def _discover_file(path)
       _globbed_files(path).each do |file|
         next if @files.member?(file)
         next if @unwatched.member?(file)
@@ -175,15 +244,24 @@ module FileWatch
         next if skip
 
         stat = File::Stat.new(file)
-        @files[file] = {
-          :size => 0,
-          :inode => inode(file,stat),
-          :create_sent => false,
-          :initial => initial
-        }
+        # let the caller build the object in its context
+        watched_file = yield(file, stat)
+
+        if discover_expired?(stat)
+          msg = "_discover_file: #{file}: skipping because it was last modified more than #{@ignore_after} seconds ago"
+          @logger.debug? && @logger.debug(msg)
+          watched_file.update(stat)
+        end
+
+        @files[file] = watched_file
       end
     end # def _discover_file
 
+    private
+    def expiry_enabled?
+      !@ignore_after.nil?
+    end
+
     private
     def _globbed_files(path)
       globbed_dirs = Dir.glob(path)
@@ -201,9 +279,19 @@ module FileWatch
       @lock.synchronize { block.call }
     end
 
+    private
+    def quit?
+      @quit_lock.synchronize { @quit }
+    end
+
+    private
+    def reset_quit
+      @quit_lock.synchronize { @quit = false }
+    end
+
     public
     def quit
-      @quit = true
+      @quit_lock.synchronize { @quit = true }
     end # def quit
   end # class Watch
 end # module FileWatch

data/lib/filewatch/yielding_tail.rb

@@ -0,0 +1,79 @@
+require 'filewatch/tail_base'
+
+module FileWatch
+  class YieldingTail
+    include TailBase
+
+    public
+    def subscribe(&block)
+      # subscribe(stat_interval = 1, discover_interval = 5, &block)
+      @watch.subscribe(@opts[:stat_interval],
+                       @opts[:discover_interval]) do |event, path|
+        case event
+        when :create, :create_initial
+          if @files.member?(path)
+            @logger.debug? && @logger.debug("#{event} for #{path}: already exists in @files")
+            next
+          end
+          if _open_file(path, event)
+            yield_read_file(path, &block)
+          end
+        when :modify
+          if !@files.member?(path)
+            @logger.debug? && @logger.debug(":modify for #{path}, does not exist in @files")
+            if _open_file(path, event)
+              yield_read_file(path, &block)
+            end
+          else
+            yield_read_file(path, &block)
+          end
+        when :delete
+          @logger.debug? && @logger.debug(":delete for: #{path} - closed and deleted from @files")
+          if @files[path]
+            yield_read_file(path, &block)
+            @files[path].close
+          end
+          @files.delete(path)
+          @statcache.delete(path)
+        when :timeout
+          @logger.debug? && @logger.debug(":timeout for: #{path} - closed and deleted from @files")
+          if (deleted = @files.delete(path))
+            deleted.close
+          end
+          @statcache.delete(path)
+        else
+          @logger.warn("unknown event type #{event} for #{path}")
+        end
+      end # @watch.subscribe
+    end # def subscribe
+
+    private
+    def yield_read_file(path, &block)
+      @buffers[path] ||= FileWatch::BufferedTokenizer.new(@opts[:delimiter])
+      delimiter_byte_size = @opts[:delimiter].bytesize
+      changed = false
+      loop do
+        begin
+          data = @files[path].sysread(32768)
+          changed = true
+          @buffers[path].extract(data).each do |line|
+            yield(path, line)
+            @sincedb[@statcache[path]] += (line.bytesize + delimiter_byte_size)
+          end
+        rescue Errno::EWOULDBLOCK, Errno::EINTR, EOFError
+          break
+        end
+      end
+
+      if changed
+        now = Time.now.to_i
+        delta = now - @sincedb_last_write
+        if delta >= @opts[:sincedb_write_interval]
+          @logger.debug? && @logger.debug("writing sincedb (delta since last write = #{delta})")
+          _sincedb_write
+          @sincedb_last_write = now
+        end
+      end
+    end
+  end # module YieldingTail
+end # module FileWatch

metadata

@@ -1,32 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: filewatch
 version: !ruby/object:Gem::Version
-  version: 0.6.7
+  version: 0.6.8
 platform: ruby
 authors:
 - Jordan Sissel
 - Pete Fritchman
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-02 00:00:00.000000000 Z
+date: 2015-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: stud
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Watch files and directories in ruby. Also supports tailing and glob file
-  patterns.
+description: Watch files and directories in ruby. Also supports tailing and glob file patterns.
 email:
 - jls@semicomplete.com
 - petef@databits.net
@@ -39,13 +38,12 @@ files:
 - lib/JRubyFileExtension.jar
 - lib/filewatch/buftok.rb
 - lib/filewatch/helper.rb
+- lib/filewatch/observing_tail.rb
 - lib/filewatch/tail.rb
+- lib/filewatch/tail_base.rb
 - lib/filewatch/watch.rb
 - lib/filewatch/winhelper.rb
-- spec/buftok_spec.rb
-- spec/tail_spec.rb
-- spec/watch_spec.rb
-- spec/winhelper_spec.rb
+- lib/filewatch/yielding_tail.rb
 - test/filewatch/tail.rb
 - test/globtail/Makefile
 - test/globtail/framework.sh
@@ -72,25 +70,25 @@ files:
 homepage: https://github.com/jordansissel/ruby-filewatch
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.6
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
 specification_version: 4
 summary: filewatch - file watching for ruby
 test_files: []

data/spec/buftok_spec.rb

@@ -1,18 +0,0 @@
-require 'filewatch/buftok'
-
-describe FileWatch::BufferedTokenizer do
-
-  context "when using the default delimiter" do
-    it "splits the lines correctly" do
-      expect(subject.extract("hello\nworld\n")).to eq ["hello", "world"]
-    end
-  end
-
-  context "when passing a custom delimiter" do
-    subject { FileWatch::BufferedTokenizer.new("\r\n") }
-
-    it "splits the lines correctly" do
-      expect(subject.extract("hello\r\nworld\r\n")).to eq ["hello", "world"]
-    end
-  end
-end

data/spec/tail_spec.rb

@@ -1,232 +0,0 @@
-require 'filewatch/tail'
-require 'stud/temporary'
-require "rbconfig"
-
-describe FileWatch::Tail do
-  before(:all) do
-    @thread_abort = Thread.abort_on_exception
-    Thread.abort_on_exception = true
-  end
-
-  after(:all) do
-    Thread.abort_on_exception = @thread_abort
-  end
-
-  let(:file_path) { f = Stud::Temporary.pathname }
-  let(:sincedb_path) { Stud::Temporary.pathname }
-
-  before :each do
-    Thread.new(subject) { sleep 0.5; subject.quit } # force the subscribe loop to exit
-  end
-
-  context "when watching a new file" do
-    subject { FileWatch::Tail.new(:sincedb_path => sincedb_path, :start_new_files_at => :beginning, :stat_interval => 0) }
-
-    before :each do
-      subject.tail(file_path)
-      File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
-    end
-
-    it "reads new lines off the file" do
-      expect { |b| subject.subscribe(&b) }.to yield_successive_args([file_path, "line1"], [file_path, "line2"])
-    end
-  end
-
-  context "when watching a file" do
-    subject { FileWatch::Tail.new(:sincedb_path => sincedb_path, :start_new_files_at => :beginning, :stat_interval => 0) }
-
-    before :each do
-      File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
-      subject.tail(file_path)
-    end
-
-    it "reads new lines off the file" do
-      expect { |b| subject.subscribe(&b) }.to yield_successive_args([file_path, "line1"], [file_path, "line2"])
-    end
-
-  end
-
-  context "when watching a CRLF file" do
-    subject { FileWatch::Tail.new(:sincedb_path => sincedb_path,
-                                  :start_new_files_at => :beginning,
-                                  :delimiter => "\r\n") }
-
-    before :each do
-      File.open(file_path, "wb") { |file|  file.write("line1\r\nline2\r\n") }
-      subject.tail(file_path)
-    end
-
-    it "reads new lines off the file" do
-      expect { |b| subject.subscribe(&b) }.to yield_successive_args([file_path, "line1"], [file_path, "line2"])
-    end
-  end
-
-  context "when a file is deleted" do
-    subject { FileWatch::Tail.new(:sincedb_path => sincedb_path, :start_new_files_at => :beginning) }
-
-    before :each do
-      File.open(file_path, "w") { |file|  file.write("line1\nline2\n") }
-      subject.tail(file_path)
-      File.unlink file_path
-    end
-
-    it "should not raise exception" do
-      Thread.new(subject) { sleep 0.1; subject.quit } # force the subscribe loop to exit
-      expect { subject.subscribe {|p,l| } }.to_not raise_exception
-    end
-  end
-
-  describe "sincedb" do
-    subject { FileWatch::Tail.new(:sincedb_path => sincedb_path, :start_new_files_at => :beginning, :stat_interval => 0) }
-
-    before :each do
-      File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
-      subject.tail(file_path)
-    end
-
-    context "when reading a new file" do
-      it "updates sincedb after subscribe" do
-        subject.subscribe {|_,_|  }
-        stat = File::Stat.new(file_path)
-        sincedb_id = subject.sincedb_record_uid(file_path,stat).join(' ')
-        expect(File.read(sincedb_path)).to eq("#{sincedb_id} #{stat.size}\n")
-      end
-    end
-
-    context "when restarting tail" do
-      before :each do
-        subject.subscribe {|_,_| }
-        sleep 0.6 # wait for tail.quit
-        subject.tail(file_path) # re-tail file
-        File.open(file_path, "ab") { |file| file.write("line3\nline4\n") }
-        Thread.new(subject) { sleep 0.5; subject.quit }
-      end
-
-      it "picks off from where it stopped" do
-        expect { |b| subject.subscribe(&b) }.to yield_successive_args([file_path, "line3"], [file_path, "line4"])
-      end
-
-      it "updates on tail.quit" do
-        subject.subscribe {|_,_| }
-        stat = File::Stat.new(file_path)
-        sincedb_id = subject.sincedb_record_uid(file_path,stat).join(' ')
-        expect(File.read(sincedb_path)).to eq("#{sincedb_id} #{stat.size}\n")
-      end
-    end
-  end
-
-  context "ingesting files bigger than 32k" do
-    let(:lineA) { "a" * 12000 }
-    let(:lineB) { "b" * 25000 }
-    let(:lineC) { "c" * 8000 }
-    subject { FileWatch::Tail.new(:sincedb_path => sincedb_path, :start_new_files_at => :beginning) }
-
-    before :each do
-      IO.write(file_path, "#{lineA}\n#{lineB}\n#{lineC}\n")
-    end
-
-    context "when restarting after stopping at the first line" do
-
-      let(:new_subject) { FileWatch::Tail.new(:sincedb_path => sincedb_path, :start_new_files_at => :beginning) }
-
-      before :each do
-        subject.tail(file_path)
-        subject.subscribe {|f, l| break if @test; @test = 1}
-        subject.sincedb_write
-        subject.quit
-        Thread.new(new_subject) { sleep 0.5; new_subject.quit } # force the subscribe loop to exit
-      end
-
-      it "should store in sincedb the position up until the first string" do
-        device, dev_major, dev_minor, pos = *IO.read(sincedb_path).split(" ").map {|n| n.to_i }
-        expect(pos).to eq(12001) # string.bytesize + "\n".bytesize
-      end
-
-      it "should read the second and third lines entirely" do
-        new_subject.tail(file_path) # re-tail file
-        expect { |b| new_subject.subscribe(&b) }.to yield_successive_args([file_path, lineB], [file_path, lineC])
-      end
-    end
-  end
-
-  context "when watching a directory" do
-
-    let(:directory) { Stud::Temporary.directory }
-    let(:file_path) { File.join(directory, "1.log") }
-
-    subject { FileWatch::Tail.new(:sincedb_path => sincedb_path, :start_new_files_at => :beginning, :stat_interval => 0) }
-
-    before :each do
-      File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
-      subject.tail(File.join(directory, "*"))
-    end
-
-    after :each do
-      FileUtils.rm_rf(directory)
-    end
-
-    it "reads new lines from the beginning" do
-      expect { |b| subject.subscribe(&b) }.to yield_successive_args([file_path, "line1"], [file_path, "line2"])
-    end
-
-    context "when a file is renamed" do
-
-      before :each do
-        expect { |b| subject.subscribe(&b) }.to yield_successive_args([file_path, "line1"], [file_path, "line2"])
-        File.rename(file_path, file_path + ".bak")
-      end
-
-      it "should not re-read the file" do
-        Thread.new(subject) { |s| sleep 1; s.quit }
-        expect { |b| subject.subscribe(&b) }.not_to yield_control
-      end
-    end
-
-    let(:new_file_path) { File.join(directory, "2.log") }
-
-    context "when a new file is later added to the directory" do
-      # Note tests in this context rely on FileWatch::Watch reading
-      # file 1.log first then 2.log and that depends on how Dir.glob is implemented
-      # in different rubies on different operating systems
-      before do
-        File.open(new_file_path, "wb") { |file|  file.write("line2.1\nline2.2\n") }
-      end
-
-      it "reads new lines from the beginning for all files" do
-        expect { |b| subject.subscribe(&b) }.to yield_successive_args([file_path, "line1"], [file_path, "line2"],
-                                                                        [new_file_path, "line2.1"], [new_file_path, "line2.2"])
-      end
-
-      context "and when the sincedb path is not given" do
-        subject { FileWatch::Tail.new(:start_new_files_at => :beginning, :stat_interval => 0) }
-
-        it "reads new lines from the beginning for all files" do
-          expect { |b| subject.subscribe(&b) }.to yield_successive_args([file_path, "line1"], [file_path, "line2"],
-                                                                        [new_file_path, "line2.1"], [new_file_path, "line2.2"])
-        end
-      end
-    end
-  end
-
-  if RbConfig::CONFIG['host_os'] !~ /mswin|mingw|cygwin/
-    context "when quiting" do
-      subject { FileWatch::Tail.new(:sincedb_path => sincedb_path, :start_new_files_at => :beginning, :stat_interval => 0) }
-
-      before :each do
-        subject.tail(file_path)
-        File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
-      end
-
-      it "closes the file handles" do
-        buffer = []
-        subject.subscribe do |path, line|
-          buffer.push([path, line])
-        end
-        subject.sincedb_write
-        subject.quit
-        lsof = `lsof -p #{Process.pid} | grep #{file_path}`
-        expect(lsof).to be_empty
-      end
-    end
-  end
-end