filewatch-ext 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ef395e982a627cd01f74e8fafd896db0c9a7b257
+  data.tar.gz: f97e919727137bab3d6f847c63f00426a655aa44
+SHA512:
+  metadata.gz: 6e181062f46b83dd8e8a131378f103fe000052923395a590ffd4b5796692505bbfff7427595c56370018a9b26d0c4b592ba2414cdcd1facb55a0a61b4405a0a2
+  data.tar.gz: 67c79b025ad906c2669f7ac176a00163705967faf6d03067317cb526bf458880ce1a98d78c79eab67ce8490d8d5713812dc4dac6eb21721acb8eec6b600c45bc

data/lib/filewatch/ext/filetail.rb

@@ -0,0 +1,132 @@
+require "filewatch/ext/tailbase"
+require "filewatch/buftok"
+
+module FileWatch::Ext
+  class FileTail < FileWatch::Ext::TailBase
+
+    public
+    def subscribe(&block)
+      # subscribe(stat_interval = 1, discover_interval = 5, &block)
+      @watch.subscribe(@opts[:stat_interval],
+                       @opts[:discover_interval]) do |event, path|
+        case event
+        when :create, :create_initial
+          if @files.member?(path)
+            @logger.debug("#{event} for #{path}: already exists in @files")
+            next
+          end
+          if _open_file(path, event)
+            _read_file(path, &block)
+          end
+        when :modify
+          _open_file(path, event) if @opts[:eof_close]
+
+          if !@files.member?(path)
+            @logger.debug(":modify for #{path}, does not exist in @files")
+            if _open_file(path, event)
+              _read_file(path, &block)
+            end
+          else
+            _read_file(path, &block)
+          end
+        when :delete
+          @logger.debug(":delete for #{path}, deleted from @files")
+
+          _read_file(path, &block) if !@opts[:eof_close]
+          _progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
+
+          @files[path].close if !@opts[:eof_close]
+          @files.delete(path)
+          @statcache.delete(path)
+        else
+          @logger.warn("unknown event type #{event} for #{path}")
+        end
+      end # @watch.subscribe
+    end # def each
+
+    private
+    def _open_file(path, event)
+      @logger.debug("_open_file: #{path}: opening")
+      begin
+        if @iswindows && defined? JRUBY_VERSION
+            @files[path] = Java::RubyFileExt::getRubyFile(path)
+        else
+            @files[path] = File.open(path)
+        end
+      rescue
+        # don't emit this message too often. if a file that we can't
+        # read is changing a lot, we'll try to open it more often,
+        # and might be spammy.
+        now = Time.now.to_i
+        if now - @lastwarn[path] > OPEN_WARN_INTERVAL
+          @logger.warn("failed to open #{path}: #{$!}")
+          @lastwarn[path] = now
+        else
+          @logger.debug("(warn supressed) failed to open #{path}: #{$!}")
+        end
+        @files.delete(path)
+        return false
+      end
+
+      stat = File::Stat.new(path)
+
+      if @iswindows
+        fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
+        inode = [fileId, stat.dev_major, stat.dev_minor]
+      else
+        inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
+      end
+
+      @statcache[path] = inode
+      if @sincedb.member?(inode)
+        last_size = @sincedb[inode][:pos]
+        @logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{stat.size}")
+        if last_size <= stat.size
+          @logger.debug("#{path}: sincedb: seeking to #{last_size}")
+          @files[path].sysseek(last_size, IO::SEEK_SET)
+        else
+          @logger.debug("#{path}: last value size is greater than current value, starting over")
+          @sincedb[inode] = {:size => stat.size, :pos => 0}
+        end
+      elsif (event == :create || event == :create_initial) && @files[path]
+        # TODO(sissel): Allow starting at beginning of the file.
+        if @opts[:start_new_files_at] == :beginning
+          @logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
+          @files[path].sysseek(0, IO::SEEK_SET)
+          @sincedb[inode] = {:size => stat.size, :pos => 0}
+        else
+          # seek to end
+          @logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
+          @files[path].sysseek(stat.size, IO::SEEK_SET)
+          @sincedb[inode] = {:size => stat.size, :pos => stat.size}
+        end
+      else
+        @logger.debug("#{path}: staying at position 0, no sincedb")
+      end
+
+      return true
+    end # def _open_file
+
+    private
+    def _read_file(path, &block)
+      # BufferedTokenizer is now in codec
+      loop do
+        begin
+          data = @files[path].sysread(32768)
+
+          yield(path, data, :log)
+
+          @sincedb[@statcache[path]][:pos] = @files[path].pos
+          _check_sincedb(false, path, &block)
+        rescue EOFError
+          _check_sincedb(true, path, &block)
+          _close_file(path) if @opts[:eof_close]
+          @logger.debug("End of file reached for #{path}")
+          break
+        rescue Errno::EWOULDBLOCK, Errno::EINTR
+          break
+        end
+      end
+    end # def _read_file
+  end # class Tail
+end # module FileWatch

data/lib/filewatch/ext/tailbase.rb

@@ -0,0 +1,179 @@
+require "filewatch/watch"
+if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
+  require "filewatch/winhelper"
+end
+require "logger"
+require "rbconfig"
+
+include Java if defined? JRUBY_VERSION
+require "JRubyFileExtension.jar" if defined? JRUBY_VERSION
+
+module FileWatch::Ext
+  class TailBase
+    # how often (in seconds) we @logger.warn a failed file open, per path.
+    OPEN_WARN_INTERVAL = ENV["FILEWATCH_OPEN_WARN_INTERVAL"] ?
+                         ENV["FILEWATCH_OPEN_WARN_INTERVAL"].to_i : 300
+
+    attr_accessor :logger
+
+    class NoSinceDBPathGiven < StandardError; end
+
+    public
+    def initialize(opts={})
+      @iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
+
+      if opts[:logger]
+        @logger = opts[:logger]
+      else
+        @logger = Logger.new(STDERR)
+        @logger.level = Logger::INFO
+      end
+      @files = {}
+      @lastwarn = Hash.new { |h, k| h[k] = 0 }
+      @watch = FileWatch::Watch.new
+      @watch.logger = @logger
+      @sincedb = {}
+      @sincedb_last_write = 0
+      @statcache = {}
+      @opts = {
+        :sincedb_write_interval => 10,
+        :stat_interval => 1,
+        :discover_interval => 5,
+        :exclude => [],
+        :start_new_files_at => :end,
+        :progressdb => false,
+        :eof_close => false,
+      }.merge(opts)
+      if !@opts.include?(:sincedb_path)
+        @opts[:sincedb_path] = File.join(ENV["HOME"], ".sincedb") if ENV.include?("HOME")
+        @opts[:sincedb_path] = ENV["SINCEDB_PATH"] if ENV.include?("SINCEDB_PATH")
+      end
+      if !@opts.include?(:sincedb_path)
+        raise NoSinceDBPathGiven.new("No HOME or SINCEDB_PATH set in environment. I need one of these set so I can keep track of the files I am following.")
+      end
+      @watch.exclude(@opts[:exclude])
+
+      _sincedb_open
+    end # def initialize
+
+    public
+    def logger=(logger)
+      @logger = logger
+      @watch.logger = logger
+    end # def logger=
+
+    public
+    def tail(path)
+      @watch.watch(path)
+    end # def tail
+
+    public
+    def subscribe(&block)
+      # to be overwritten
+    end # def each
+
+    protected
+    def _open_file(path, event)
+      # to be overwritten
+    end # def _open_file
+
+    protected
+    def _read_file(path, &block)
+      # to be overwritten
+    end # def _read_file
+
+    private
+    def _close_file(path)
+      @files[path].close
+    end
+
+    protected
+    def _check_sincedb(eof, path, &block)
+      now = Time.now.to_i
+      delta = now - @sincedb_last_write
+      if eof || delta >= @opts[:sincedb_write_interval]
+        @logger.debug("writing sincedb (delta since last write = #{delta})") if @logger.debug?
+        _sincedb_write
+        _progressdb_write(path, &block) if @opts[:progressdb]
+        @sincedb_last_write = now
+      end
+    end
+
+    public
+    def sincedb_write(reason=nil)
+      @logger.debug("caller requested sincedb write (#{reason})")
+      _sincedb_write
+    end
+
+    protected
+    def _sincedb_open
+      path = @opts[:sincedb_path]
+      begin
+        db = File.open(path)
+      rescue
+        @logger.debug("_sincedb_open: #{path}: #{$!}")
+        return
+      end
+
+      @logger.debug("_sincedb_open: reading from #{path}")
+      db.each do |line|
+        ino, dev_major, dev_minor, size, pos = line.split(" ", 5)
+        inode = [ino, dev_major.to_i, dev_minor.to_i]
+        @logger.debug("_sincedb_open: setting #{inode.inspect} to #{pos.to_i}")
+        @sincedb[inode] = {:size => size.to_i, :pos => pos.to_i}
+      end
+    end # def _sincedb_open
+
+    protected
+    def _sincedb_write
+      path = @opts[:sincedb_path]
+      tmp = "#{path}.new"
+      begin
+        db = File.open(tmp, "w")
+      rescue => e
+        @logger.warn("_sincedb_write failed: #{tmp}: #{e}")
+        return
+      end
+
+      @sincedb.each do |inode, meta|
+        db.puts([inode, meta[:size], meta[:pos]].flatten.join(" "))
+      end
+      db.close
+
+      begin
+        File.rename(tmp, path)
+      rescue => e
+        @logger.warn("_sincedb_write rename/sync failed: #{tmp} -> #{path}: #{e}")
+      end
+    end # def _sincedb_write
+
+    protected
+    def _sincedb_delete(path)
+      inode = @statcache[path]
+      @sincedb.delete(inode)
+      _sincedb_write
+    end
+
+    protected
+    def _progressdb_write(path, &block)
+      @sincedb.each do |inode, meta|
+        line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
+        yield(path, line, :progressdb)
+      end
+    end
+
+    protected
+    def _progressdb_delete(path, &block)
+      inode = @statcache[path]
+      meta = @sincedb[inode]
+
+      line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
+      yield(path, line, :progressdb_del)
+    end
+
+    public
+    def quit
+      @watch.quit
+    end # def quit
+  end # class Tail
+end # module FileWatch

data/lib/filewatch/ext/xlstail.rb

@@ -0,0 +1,149 @@
+require "filewatch/ext/tailbase"
+require "spreadsheet"
+
+module FileWatch::Ext
+  class XlsTail < FileWatch::Ext::TailBase
+
+    public
+    def initialize(opts={})
+      @pos = {}
+      super
+    end
+
+    public
+    def subscribe(&block)
+      # subscribe(stat_interval = 1, discover_interval = 5, &block)
+      @watch.subscribe(@opts[:stat_interval],
+                       @opts[:discover_interval]) do |event, path|
+        case event
+        when :create, :create_initial
+          if @files.member?(path)
+            @logger.debug("#{event} for #{path}: already exists in @files")
+            next
+          end
+          if _open_file(path, event)
+            _read_file(path, &block)
+          end
+        when :modify
+          @logger.debug(":modify for #{path}")
+        when :delete
+          @logger.debug(":delete for #{path}, deleted from @files")
+
+          _progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
+
+          @files.delete(path)
+          @statcache.delete(path)
+        else
+          @logger.warn("unknown event type #{event} for #{path}")
+        end
+      end # @watch.subscribe
+    end # def each
+
+    private
+    def get_size(path)
+      workbook = Spreadsheet.open(path)
+      pos = 0
+      workbook.worksheets.each do |worksheet|
+        pos += worksheet.rows.length
+      end
+      @size = pos
+    end
+
+    private
+    def _open_file(path, event)
+      @logger.debug("_open_file: #{path}: opening")
+      begin
+        #Spreadsheet.client_encoding = 'UTF-8'
+        @files[path] = Spreadsheet.open(path)
+      rescue
+        # don't emit this message too often. if a file that we can't
+        # read is changing a lot, we'll try to open it more often,
+        # and might be spammy.
+        now = Time.now.to_i
+        if now - @lastwarn[path] > OPEN_WARN_INTERVAL
+          @logger.warn("failed to open #{path}: #{$!}")
+          @lastwarn[path] = now
+        else
+          @logger.debug("(warn supressed) failed to open #{path}: #{$!}")
+        end
+        #@files.delete(path)
+        return false
+      end
+
+      stat = File::Stat.new(path)
+      size = get_size(path)
+
+      if @iswindows
+        fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
+        inode = [fileId, stat.dev_major, stat.dev_minor]
+      else
+        inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
+      end
+
+      @statcache[path] = inode
+
+      if @sincedb.member?(inode)
+        last_size = @sincedb[inode][:pos]
+        @logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{size}")
+        if last_size <= size
+          @logger.debug("#{path}: sincedb: seeking to #{last_size}")
+          @pos[path] = last_size
+        else
+          @logger.debug("#{path}: last value size is greater than current value, starting over")
+          @sincedb[inode] = {:size => size, :pos => 0}
+        end
+      elsif event == :create_initial && @files[path]
+        # TODO(sissel): Allow starting at beginning of the file.
+        if @opts[:start_new_files_at] == :beginning
+          @logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
+          @pos[path] = 0
+          @sincedb[inode] = {:size => size, :pos => 0}
+        else
+          # seek to end
+          @logger.debug("#{path}: initial create, no sincedb, seeking to end #{size}")
+          @pos[path] = size
+          @sincedb[inode] = {:size => size, :pos => size}
+        end
+      else
+        @logger.debug("#{path}: staying at position 0, no sincedb")
+      end
+
+      return true
+    end # def _open_file
+
+    private
+    def _read_file(path, &block)
+      changed = false
+      pos = 0
+
+      worksheets = @files[path].worksheets
+
+      worksheets.each_with_index do |worksheet, index_sheet|
+
+        worksheet.each_with_index do |row, index_row|
+          pos += 1
+
+          if pos > @pos[path]
+            changed = true
+
+            sheet_name = worksheet.name.empty? ? "Sheet#{index_sheet+1}" : worksheet.name
+
+            if pos == @size #end of file is reached
+              data = {:row => row, :wsname => sheet_name, :eof => true}
+            else
+              data = {:row => row, :wsname => sheet_name, :eof => false}
+            end
+
+            yield(path, data, :log)
+
+            @pos[path] = pos
+            @sincedb[@statcache[path]][:pos] = pos
+            _check_sincedb(false, path, &block)
+          end
+        end
+      end
+      _check_sincedb(true, path, &block) if changed
+    end # def _read_file
+
+  end # class Tail
+end # module FileWatch

data/lib/filewatch/ext/xlsxtail.rb

@@ -0,0 +1,147 @@
+require "filewatch/ext/tailbase"
+require "simple_xlsx_reader"
+
+module FileWatch::Ext
+  class XlsxTail < FileWatch::Ext::TailBase
+
+    public
+    def initialize(opts={})
+      @pos = {}
+      super
+    end
+
+    public
+    def subscribe(&block)
+      # subscribe(stat_interval = 1, discover_interval = 5, &block)
+      @watch.subscribe(@opts[:stat_interval],
+                       @opts[:discover_interval]) do |event, path|
+        case event
+        when :create, :create_initial
+          if @files.member?(path)
+            @logger.debug("#{event} for #{path}: already exists in @files")
+            next
+          end
+          if _open_file(path, event)
+            _read_file(path, &block)
+          end
+        when :modify
+          @logger.debug(":modify for #{path}")
+        when :delete
+          @logger.debug(":delete for #{path}, deleted from @files")
+
+          _progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
+
+          @files.delete(path)
+          @statcache.delete(path)
+        else
+          @logger.warn("unknown event type #{event} for #{path}")
+        end
+      end # @watch.subscribe
+    end # def each
+
+    private
+    def get_size(path)
+      doc = SimpleXlsxReader.open(path)
+      pos = 0
+      doc.sheets.each do |worksheet|
+        pos += worksheet.rows.length
+      end
+      @size = pos
+    end
+
+    private
+    def _open_file(path, event)
+      @logger.debug("_open_file: #{path}: opening")
+      begin
+        @files[path] = SimpleXlsxReader.open(path)
+      rescue
+        # don't emit this message too often. if a file that we can't
+        # read is changing a lot, we'll try to open it more often,
+        # and might be spammy.
+        now = Time.now.to_i
+        if now - @lastwarn[path] > OPEN_WARN_INTERVAL
+          @logger.warn("failed to open #{path}: #{$!}")
+          @lastwarn[path] = now
+        else
+          @logger.debug("(warn supressed) failed to open #{path}: #{$!}")
+        end
+        #@files.delete(path)
+        return false
+      end
+
+      stat = File::Stat.new(path)
+      size = get_size(path)
+
+      if @iswindows
+        fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
+        inode = [fileId, stat.dev_major, stat.dev_minor]
+      else
+        inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
+      end
+
+      @statcache[path] = inode
+
+      if @sincedb.member?(inode)
+        last_size = @sincedb[inode][:pos]
+        @logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{size}")
+        if last_size <= size
+          @logger.debug("#{path}: sincedb: seeking to #{last_size}")
+          @pos[path] = last_size
+        else
+          @logger.debug("#{path}: last value size is greater than current value, starting over")
+          @sincedb[inode] = {:size => size, :pos => 0}
+        end
+      elsif event == :create_initial && @files[path]
+        # TODO(sissel): Allow starting at beginning of the file.
+        if @opts[:start_new_files_at] == :beginning
+          @logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
+          @pos[path] = 0
+          @sincedb[inode] = {:size => size, :pos => 0}
+        else
+          # seek to end
+          @logger.debug("#{path}: initial create, no sincedb, seeking to end #{size}")
+          @pos[path] = size
+          @sincedb[inode] = {:size => size, :pos => size}
+        end
+      else
+        @logger.debug("#{path}: staying at position 0, no sincedb")
+      end
+
+      return true
+    end # def _open_file
+
+    private
+    def _read_file(path, &block)
+      changed = false
+      pos = 0
+
+      worksheets = @files[path].sheets
+
+      worksheets.each_with_index do |worksheet, index_sheet|
+
+        worksheet.rows.each_with_index do |row, index_row|
+          pos += 1
+
+          if pos > @pos[path]
+            changed = true
+            sheet_name = worksheet.name.empty? ? "Sheet#{index_sheet+1}" : worksheet.name
+
+            if pos == @size #end of file is reached
+              data = {:row => row, :wsname => sheet_name, :eof => true}
+            else
+              data = {:row => row, :wsname => sheet_name, :eof => false}
+            end
+
+            yield(path, data, :log)
+
+            @pos[path] = pos
+            @sincedb[@statcache[path]][:pos] = pos
+            _check_sincedb(false, path, &block)
+          end
+        end
+      end
+      _check_sincedb(true, path, &block) if changed
+    end # def _read_file
+
+  end # class Tail
+end # module FileWatch

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: filewatch-ext
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dietmar Duft
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-03-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: filewatch
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: simple_xlsx_reader
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.8
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.8
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: axlsx
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+  prerelease: false
+  type: :runtime
+description: Extensions for ruby-filewatcher, like xls, xlsx
+email:
+- dietmar@signifydata.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/JRubyFileExtension.jar
+- lib/filewatch/ext/tailbase.rb
+- lib/filewatch/ext/filetail.rb
+- lib/filewatch/ext/xlsxtail.rb
+- lib/filewatch/ext/xlstail.rb
+homepage: https://bitbucket.org/signify/ruby-filewatch-ext
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: filewatch extensions
+test_files: []