filewatch-ext 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7aaa9136a78754ff3bf796e41f7f90ebf8d85a0b
-  data.tar.gz: f562eaacb52a89b0d0663ec0943cea657a0e606a
+  metadata.gz: cc02b1e8e969759f62b9962cea5a75e90166d878
+  data.tar.gz: c574c0b6b2c0afa7184bf9d83de1cfbece399298
 SHA512:
-  metadata.gz: 6b5fde10a6257ba99fcdaf62ac89d3a4f2536f9b7cb556ca42b850ae44b90622e6c6a5a2fa57702d12cf5703eeb5acbeca4e9c783b7eca8a84ad3ce04b6819fc
-  data.tar.gz: d03e3131588f827ab474e58c6f898f9e81bd9edf7d91ba9c8e184207baa612a1e4af0b633602d53d2fef5335ed29f00a62fe65f87d7ca34a044ac93666042487
+  metadata.gz: 6d53563ad238de688ebe97921dcb51ebab42301d27772929a37ca207582b9a5b4300b8e1fefddc30171d9572deeb44371a506f8cd68c3f7aaccb1a0b1273bfe0
+  data.tar.gz: cdb418c8ede09c47ad56370f460789232c85353e07bdf0ed3e2adc124b830567547a9e28fef915caea1f4f013410014417e30378fce6dc1f9599149f002e3723

data/lib/filewatch/ext/progress_tail.rb

@@ -0,0 +1,61 @@
+require 'filewatch/observing_tail'
+
+module FileWatch
+  module Ext
+    class ProgressTail < FileWatch::ObservingTail
+
+      private
+      def observe_read_file(watched_file, listener)
+        changed = false
+        loop do
+          begin
+            data = watched_file.file_read(32768)
+            changed = true
+            watched_file.buffer_extract(data).each do |line|
+              listener.accept(line)
+              @sincedb[watched_file.inode] += (line.bytesize + @delimiter_byte_size)
+            end
+            # watched_file bytes_read tracks the sincedb entry
+            # see TODO in watch.rb
+            watched_file.update_bytes_read(@sincedb[watched_file.inode])
+
+            _progressdb_send(watched_file, listener)
+          rescue EOFError
+            listener.eof
+            watched_file.unwatch if @opts[:eof_close]
+            break
+          rescue Errno::EWOULDBLOCK, Errno::EINTR
+            listener.error
+            break
+          rescue => e
+            @logger.debug? && @logger.debug("observe_read_file: general error reading #{watched_file.path} - error: #{e.inspect}")
+            listener.error
+            break
+          end
+        end
+
+        if changed
+          now = Time.now.to_i
+          delta = now - @sincedb_last_write
+          if delta >= @opts[:sincedb_write_interval]
+            @logger.debug? && @logger.debug("writing sincedb (delta since last write = #{delta})")
+            _sincedb_write
+            @sincedb_last_write = now
+          end
+        end
+      end # def _read_file
+
+
+      private
+      def _progressdb_send watched_file, listener
+        path = watched_file.path
+        sincedb_key = watched_file.inode
+        stat = watched_file.filestat
+
+        line = [sincedb_key, stat.size, @sincedb[sincedb_key]].flatten.join(" ")
+
+        listener.progress(line)
+      end
+    end # module ProgressTail
+  end # module Ext
+end # module FileWatch

data/lib/filewatch/ext/tail.rb

@@ -0,0 +1,22 @@
+require "filewatch/ext/progress_tail"
+require "forwardable"
+
+module FileWatch
+	module Ext
+		class Tail
+			extend Forwardable
+
+			def_delegators :@target, :tail, :logger=, :subscribe, :sincedb_record_uid, :sincedb_write, :quit, :close_file
+
+			attr_writer :target
+
+			def self.new_observing_progress(opts = {})
+				new({}, ProgressTail.new(opts))
+			end
+
+		    def initialize(opts = {}, target = YieldingTail.new(opts))
+		      	@target = target
+		    end
+		end
+	end
+end

metadata

@@ -1,35 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: filewatch-ext
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Signify
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-25 00:00:00.000000000 Z
+date: 2016-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: filewatch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.5
+        version: 0.8.1
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.1
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.5
+        version: '0'
 description: Extensions for ruby-filewatcher
 email:
 - dietmar@signifydata.com
@@ -37,10 +51,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/JRubyFileExtension.jar
-- lib/filewatch/ext/filetail.rb
-- lib/filewatch/ext/tailbase.rb
-- lib/filewatch/winhelper.rb
+- lib/filewatch/ext/progress_tail.rb
+- lib/filewatch/ext/tail.rb
 homepage: https://bitbucket.org/signify/ruby-filewatch-ext
 licenses:
 - MIT

data/lib/filewatch/ext/filetail.rb

@@ -1,135 +0,0 @@
-require "filewatch/ext/tailbase"
-require "filewatch/buftok"
-
-module FileWatch::Ext
-  class FileTail < FileWatch::Ext::TailBase
-
-    public
-    def subscribe(&block)
-      # subscribe(stat_interval = 1, discover_interval = 5, &block)
-      @watch.subscribe(@opts[:stat_interval],
-                       @opts[:discover_interval]) do |event, path|
-        case event
-        when :create, :create_initial
-          if @files.member?(path)
-            @logger.debug("#{event} for #{path}: already exists in @files")
-            next
-          end
-          if _open_file(path, event)
-            _read_file(path, &block)
-          end
-        when :modify
-          _open_file(path, event) if @opts[:eof_close]
-
-          if !@files.member?(path)
-            @logger.debug(":modify for #{path}, does not exist in @files")
-            if _open_file(path, event)
-              _read_file(path, &block)
-            end
-          else
-            _read_file(path, &block)
-          end
-        when :delete
-          @logger.debug(":delete for #{path}, deleted from @files")
-
-          _read_file(path, &block) if !@opts[:eof_close]
-          _progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
-
-          @files[path].close if !@opts[:eof_close]
-          @files.delete(path)
-          _sincedb_delete(path)
-          @statcache.delete(path)
-        else
-          @logger.warn("unknown event type #{event} for #{path}")
-        end
-      end # @watch.subscribe
-    end # def each
-
-    private
-    def _open_file(path, event)
-      @logger.debug("_open_file: #{path}: opening")
-      begin
-        if @iswindows && defined? JRUBY_VERSION
-            @files[path] = Java::RubyFileExt::getRubyFile(path)
-        else
-            @files[path] = File.open(path)
-        end
-      rescue
-        # don't emit this message too often. if a file that we can't
-        # read is changing a lot, we'll try to open it more often,
-        # and might be spammy.
-        now = Time.now.to_i
-        if now - @lastwarn[path] > OPEN_WARN_INTERVAL
-          @logger.warn("failed to open #{path}: #{$!}")
-          @lastwarn[path] = now
-        else
-          @logger.debug("(warn supressed) failed to open #{path}: #{$!}")
-        end
-        @files.delete(path)
-        return false
-      end
-
-      stat = File::Stat.new(path)
-
-      if @iswindows
-        fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
-        inode = [fileId, stat.dev_major, stat.dev_minor]
-      else
-        inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
-      end
-
-      @statcache[path] = inode
-      if @sincedb.member?(inode)
-        last_size = @sincedb[inode][:pos]
-        @logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{stat.size}")
-        if last_size <= stat.size
-          @logger.debug("#{path}: sincedb: seeking to #{last_size}")
-          @files[path].sysseek(last_size, IO::SEEK_SET)
-        else
-          @logger.debug("#{path}: last value size is greater than current value, starting over")
-          @sincedb[inode] = {:size => stat.size, :pos => 0}
-        end
-      elsif (event == :create || event == :create_initial) && @files[path]
-        # TODO(sissel): Allow starting at beginning of the file.
-        if @opts[:start_new_files_at] == :beginning
-          @logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
-          @files[path].sysseek(0, IO::SEEK_SET)
-          @sincedb[inode] = {:size => stat.size, :pos => 0}
-        else
-          # seek to end
-          @logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
-          @files[path].sysseek(stat.size, IO::SEEK_SET)
-          @sincedb[inode] = {:size => stat.size, :pos => stat.size}
-        end
-      else
-        @logger.debug("#{path}: staying at position 0, no sincedb")
-      end
-
-      return true
-    end # def _open_file
-
-    private
-    def _read_file(path, &block)
-      # BufferedTokenizer is now in codec
-
-      changed = false
-      loop do
-        begin
-          data = @files[path].sysread(32768)
-          changed = true
-          yield(path, data, :log)
-
-          @sincedb[@statcache[path]][:pos] = @files[path].pos
-          _check_sincedb(false, path, &block)
-        rescue EOFError
-          _check_sincedb(true, path, &block) if changed
-          _close_file(path) if @opts[:eof_close]
-          @logger.debug("End of file reached for #{path}")
-          break
-        rescue Errno::EWOULDBLOCK, Errno::EINTR
-          break
-        end
-      end
-    end # def _read_file
-  end # class Tail
-end # module FileWatch

data/lib/filewatch/ext/tailbase.rb

@@ -1,189 +0,0 @@
-require "filewatch/watch"
-if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
-  require "filewatch/winhelper"
-end
-require "logger"
-require "rbconfig"
-
-include Java if defined? JRUBY_VERSION
-require "JRubyFileExtension.jar" if defined? JRUBY_VERSION
-
-module FileWatch::Ext
-  class TailBase
-    # how often (in seconds) we @logger.warn a failed file open, per path.
-    OPEN_WARN_INTERVAL = ENV["FILEWATCH_OPEN_WARN_INTERVAL"] ?
-                         ENV["FILEWATCH_OPEN_WARN_INTERVAL"].to_i : 300
-
-    attr_accessor :logger
-
-    class NoSinceDBPathGiven < StandardError; end
-
-    public
-    def initialize(opts={})
-      @iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
-
-      if opts[:logger]
-        @logger = opts[:logger]
-      else
-        @logger = Logger.new(STDERR)
-        @logger.level = Logger::INFO
-      end
-      @files = {}
-      @lastwarn = Hash.new { |h, k| h[k] = 0 }
-      @watch = FileWatch::Watch.new
-      @watch.logger = @logger
-      @sincedb = {}
-      @sincedb_last_write = 0
-      @statcache = {}
-      @opts = {
-        :sincedb_write_interval => 10,
-        :stat_interval => 1,
-        :discover_interval => 5,
-        :exclude => [],
-        :start_new_files_at => :end,
-        :progressdb => false,
-        :eof_close => false,
-      }.merge(opts)
-      if !@opts.include?(:sincedb_path)
-        @opts[:sincedb_path] = File.join(ENV["HOME"], ".sincedb") if ENV.include?("HOME")
-        @opts[:sincedb_path] = ENV["SINCEDB_PATH"] if ENV.include?("SINCEDB_PATH")
-      end
-      if !@opts.include?(:sincedb_path)
-        raise NoSinceDBPathGiven.new("No HOME or SINCEDB_PATH set in environment. I need one of these set so I can keep track of the files I am following.")
-      end
-      @watch.exclude(@opts[:exclude])
-
-      _sincedb_open
-    end # def initialize
-
-    public
-    def logger=(logger)
-      @logger = logger
-      @watch.logger = logger
-    end # def logger=
-
-    public
-    def tail(path)
-      @watch.watch(path)
-    end # def tail
-
-    public
-    def subscribe(&block)
-      # to be overwritten
-    end # def each
-
-    protected
-    def _open_file(path, event)
-      # to be overwritten
-    end # def _open_file
-
-    protected
-    def _read_file(path, &block)
-      # to be overwritten
-    end # def _read_file
-
-    private
-    def _close_file(path)
-      @files[path].close
-    end
-
-    protected
-    def _check_sincedb(eof, path, &block)
-      now = Time.now.to_i
-      delta = now - @sincedb_last_write
-      if eof || delta >= @opts[:sincedb_write_interval]
-        @logger.debug("writing sincedb (delta since last write = #{delta})") if @logger.debug?
-        _sincedb_write
-        _progressdb_write(path, eof, &block) if @opts[:progressdb]
-        @sincedb_last_write = now
-      end
-    end
-
-    public
-    def sincedb_write(reason=nil)
-      @logger.debug("caller requested sincedb write (#{reason})")
-      _sincedb_write
-    end
-
-    protected
-    def _sincedb_open
-      path = @opts[:sincedb_path]
-      begin
-        db = File.open(path)
-      rescue
-        @logger.debug("_sincedb_open: #{path}: #{$!}")
-        return
-      end
-
-      @logger.debug("_sincedb_open: reading from #{path}")
-      db.each do |line|
-        ino, dev_major, dev_minor, size, pos = line.split(" ", 5)
-        inode = [ino, dev_major.to_i, dev_minor.to_i]
-        @logger.debug("_sincedb_open: setting #{inode.inspect} to #{pos.to_i}")
-        @sincedb[inode] = {:size => size.to_i, :pos => pos.to_i}
-      end
-    end # def _sincedb_open
-
-    protected
-    def _sincedb_write
-      path = @opts[:sincedb_path]
-      tmp = "#{path}.new"
-      begin
-        db = File.open(tmp, "w")
-      rescue => e
-        @logger.warn("_sincedb_write failed: #{tmp}: #{e}")
-        return
-      end
-
-      @sincedb.each do |inode, meta|
-        db.puts([inode, meta[:size], meta[:pos]].flatten.join(" "))
-      end
-      db.close
-
-      begin
-        File.rename(tmp, path)
-      rescue => e
-        @logger.warn("_sincedb_write rename/sync failed: #{tmp} -> #{path}: #{e}")
-      end
-    end # def _sincedb_write
-
-    protected
-    def _sincedb_delete(path)
-      inode = @statcache[path]
-      @sincedb.delete(inode)
-      _sincedb_write
-    end
-
-    protected
-    def _progressdb_write(path, eof, &block)
-      if eof
-        inode = @statcache[path]
-        meta = @sincedb[@statcache[path]]
-        line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
-        yield(path, line, :progressdb)
-      else
-        @statcache.each do |path, inode|
-          meta = @sincedb[inode]
-          if meta[:size] != meta[:pos]
-            line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
-            yield(path, line, :progressdb)
-          end
-        end
-      end
-    end
-
-    protected
-    def _progressdb_delete(path, &block)
-      inode = @statcache[path]
-      meta = @sincedb[inode]
-
-      line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
-      yield(path, line, :progressdb_del)
-    end
-
-    public
-    def quit
-      @watch.quit
-    end # def quit
-  end # class Tail
-end # module FileWatch

data/lib/filewatch/winhelper.rb

@@ -1,70 +0,0 @@
-require "ffi"
-	
-module Winhelper
-	extend FFI::Library
-	
-	ffi_lib 'kernel32'
-	ffi_convention :stdcall
-	class FileTime < FFI::Struct
-		layout :lowDateTime, :uint,
-		:highDateTime, :uint
-	end
-	
-	#http://msdn.microsoft.com/en-us/library/windows/desktop/aa363788(v=vs.85).aspx
-	class FileInformation < FFI::Struct
-		def initialize()
-			createTime = FileTime.new
-			lastAccessTime = FileTime.new
-			lastWriteTime = FileTime.new
-		end
-		
-		layout :fileAttributes, :uint, #DWORD    dwFileAttributes;
-		:createTime, FileTime,  #FILETIME ftCreationTime;
-		:lastAccessTime, FileTime, #FILETIME ftLastAccessTime;
-		:lastWriteTime, FileTime, #FILETIME ftLastWriteTime;
-		:volumeSerialNumber, :uint, #DWORD    dwVolumeSerialNumber;
-		:fileSizeHigh, :uint, #DWORD    nFileSizeHigh;
-		:fileSizeLow, :uint, #DWORD    nFileSizeLow;
-		:numberOfLinks, :uint, #DWORD    nNumberOfLinks;
-		:fileIndexHigh, :uint, #DWORD    nFileIndexHigh;
-		:fileIndexLow, :uint #DWORD    nFileIndexLow;
-	end
-		
-	
-	#http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
-	#HANDLE WINAPI CreateFile(_In_ LPCTSTR lpFileName,_In_  DWORD dwDesiredAccess,_In_ DWORD dwShareMode,
-	#						_In_opt_  LPSECURITY_ATTRIBUTES lpSecurityAttributes,_In_  DWORD dwCreationDisposition,
-	#						_In_      DWORD dwFlagsAndAttributes,_In_opt_  HANDLE hTemplateFile);
-	attach_function :GetOpenFileHandle, :CreateFileA, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer	
-	
-	#http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952(v=vs.85).aspx
-	#BOOL WINAPI GetFileInformationByHandle(_In_   HANDLE hFile,_Out_  LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
-	attach_function :GetFileInformationByHandle, [:pointer, :pointer], :int
-	
-	attach_function :CloseHandle, [:pointer], :int
-	
-	
-	def self.GetWindowsUniqueFileIdentifier(path)
-		handle = GetOpenFileHandle(path, 0, 7, nil, 3, 128, nil)
-		fileInfo = Winhelper::FileInformation.new
-		success = GetFileInformationByHandle(handle, fileInfo)
-		CloseHandle(handle)
-		if  success == 1
-			#args = [
-			#		fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow], 
-			#		fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow]
-			#	]
-			#p "Information: %u %u %u %u %u %u %u " % args
-			#this is only guaranteed on NTFS, for ReFS on windows 2012, GetFileInformationByHandleEx should be used with FILE_ID_INFO, which returns a 128 bit identifier
-			return "#{fileInfo[:volumeSerialNumber]}-#{fileInfo[:fileIndexLow]}-#{fileInfo[:fileIndexHigh]}"
-		else
-			#p "cannot retrieve file information, returning path"
-			return path;
-		end
-	end
-end
-
-#fileId = Winhelper.GetWindowsUniqueFileIdentifier('C:\inetpub\logs\LogFiles\W3SVC1\u_ex1fdsadfsadfasdf30612.log')
-#p "FileId: " + fileId
-#p "outside function, sleeping"
-#sleep(10)