rbbt-util 1.1.0 → 1.2.1

data/lib/rbbt-util.rb

@@ -4,6 +4,7 @@ require 'rbbt/util/cmd'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/filecache'
 require 'rbbt/util/tsv'
+require 'rbbt/util/bed'
 require 'rbbt/util/cachehelper'
 require 'rbbt/util/misc'
 
@@ -11,3 +12,4 @@ FileCache.cachedir = Rbbt.cachedir
 Open.cachedir      = File.join(Rbbt.cachedir, 'open-remote/')
 TmpFile.tmpdir     = File.join(Rbbt.tmpdir)
 TSV.cachedir       = File.join(Rbbt.cachedir, 'tsv_cache')
+Bed.cachedir       = File.join(Rbbt.cachedir, 'bed_cache')

data/lib/rbbt.rb

@@ -13,3 +13,4 @@ module Rbbt
   self.load_cfg(%w(tmpdir cachedir datadir))
 end
 
+

data/lib/rbbt/util/R.rb

@@ -0,0 +1,24 @@
+require 'rbbt/util/cmd'
+
+module R
+
+  LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'../../../share/lib/R')
+  UTIL    = File.join(LIB_DIR, 'util.R')
+
+  def self.run(command, options = {})
+    cmd = "source('#{UTIL}');\n"
+    case
+    when IO === command
+      cmd << command.read
+    when File.exists?(command)
+      cmd << File.open(command, 'r') do |f| f.read end
+    else
+      cmd << command
+    end
+
+    Log.debug "R Script:\n#{ cmd }"
+
+    CMD.cmd('R --vanilla --slave', options.merge(:in => cmd))
+  end
+
+end

data/lib/rbbt/util/bed.rb

@@ -0,0 +1,325 @@
+require 'rbbt/util/tsv'
+require 'rbbt/util/misc'
+require 'rbbt/util/log'
+require 'yaml'
+
+class Bed
+
+  class Entry < Struct.new( :value, :start, :end, :overlap); end
+
+  class FixWidthTable
+    SERIALIZER = Marshal
+    def self.serialize(entry)
+      SERIALIZER.dump(entry)
+    end
+
+    def self.deserialise(entry)
+      SERIALIZER.load(entry)
+    end
+
+    def self.format(entry, index, record_size)
+      data = serialize(entry)
+      padding = record_size - data.length
+      [index, data + "\0" * padding].pack("la#{record_size}")
+    end
+
+    def self.index(format)
+      format.unpack("s").first
+    end
+
+    def self.unformat(format, record_size)
+      index, data = format.unpack("la#{record_size}")
+      deserialise(data)
+    end
+
+    def self.get_record_size(entries)
+      max = 0
+      entries.each do |entry|
+        size = serialize(entry).length
+        max  = size if size > max
+      end
+
+      max
+    end
+
+    attr_accessor :size, :range
+    def initialize(file, record_size = nil, range = false, rewrite = false)
+      @filename = file
+      @range = range
+
+      if rewrite or not File.exists? file
+        Log.debug("Opening FixWidthTable in #{ file } writing. Record size: #{record_size}")
+        @file = File.open(@filename, 'wb')
+        @record_size = record_size
+        @file.write [record_size].pack("L")
+        @file.write [@range ? 1 : 0 ].pack("C")
+        @size = 0
+      else
+        Log.debug("Opening FixWidthTable in #{ file } for reading")
+        @file        = File.open(@filename, 'rb')
+        @record_size = @file.read(4).unpack("L").first
+        @range       = @file.read(1).unpack("C").first == 1
+        @size        = (File.size(@filename) - 5) / (@record_size + 4)
+        Log.debug("Record size #{@record_size}")
+      end
+    end
+
+    def length
+      @size / @record_size
+    end
+
+    def read
+      @file.close 
+      @file = File.open(@filename, 'rb')
+    end
+
+    def add(entry, index)
+      @size += 1
+      format = FixWidthTable.format(entry, index, @record_size)
+      @file.write format
+    end
+
+    def index(index)
+      return nil if index < 0 or index >= size
+      @file.seek(5 + (@record_size + 4) * index, IO::SEEK_SET)
+
+      format = @file.read(@record_size + 4)
+      FixWidthTable.index(format)
+    end
+
+    def [](index)
+      return nil if index < 0 or index >= size
+      @file.seek(5 + (@record_size + 4) * index, IO::SEEK_SET)
+
+      format = @file.read(@record_size + 4)
+      FixWidthTable.unformat(format, @record_size)
+    end
+
+    def closest(pos)
+      upper = size - 1
+      lower = 0
+
+      return -1 if upper < lower
+
+      while(upper >= lower) do
+        idx = lower + (upper - lower) / 2
+        comp = pos <=> index(idx)
+
+        if comp == 0
+          break 
+        elsif comp > 0
+          lower = idx + 1
+        else
+          upper = idx - 1
+        end
+      end
+
+      if index(idx) > pos
+        idx = idx - 1
+      end
+
+      idx
+    end
+
+
+  end
+
+  #{{{ Persistence
+
+  CACHEDIR="/tmp/bed_persistent_cache"
+  FileUtils.mkdir CACHEDIR unless File.exist? CACHEDIR
+
+  def self.cachedir=(cachedir)
+    CACHEDIR.replace cachedir
+    FileUtils.mkdir_p CACHEDIR unless File.exist? CACHEDIR
+  end
+
+  def self.cachedir
+    CACHEDIR
+  end
+
+  def self.get_persistence_file(file, prefix, options = {})
+    File.join(CACHEDIR, prefix.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, options].inspect))
+  end
+
+
+  attr_accessor :index, :range, :persistence_file
+  def initialize(tsv, options = {})
+    options = Misc.add_defaults options, :range => nil, :key => 0, :value => 1, :persistence => false, :persistence_file => nil, :tsv => {}
+
+    options[:persistence] = true if options[:persistence].nil? and options[:persistence_file]
+
+    filename = nil
+    case
+    when TSV === tsv
+      filename = tsv.filename
+    when (String === tsv and File.exists? tsv.sub(/#.*/,''))
+      filename = tsv
+    else 
+      filename = "None"
+    end
+
+
+    if options[:range]
+      options[:key]   = options[:range].first
+      options[:value] = [options[:value], options[:range].last]
+      @range = true
+    else
+      @range = false
+    end
+
+    if options[:persistence] and options[:persistence_file].nil?
+      options[:persistence_file] = Bed.get_persistence_file(filename, (options[:range].nil? ? "Point:#{filename}" : "Range:#{filename}"),  options)
+    end
+
+    @persistence_file = options[:persistence_file]
+
+    if @persistence_file and File.exists?(@persistence_file)
+      Log.low("Loading Persistence Bed File: #{ @persistence_file }")
+
+      @index = FixWidthTable.new @persistence_file
+      @range = @index.range
+      return
+    end
+
+    tsv = TSV.new(tsv, options[:tsv]) unless TSV === tsv
+
+    @index = []
+    entry = nil
+    tsv.through options[:key], options[:value] do |key, values|
+      if @range
+        entry = Entry.new(values[0], key.to_i, values[1].to_i,  nil)
+      else
+        entry = Entry.new(values[0], key.to_i, nil, nil)
+      end
+      @index << entry
+    end
+
+    @index.sort!{|a,b| a.start <=> b.start}
+
+    if range
+      latest = []
+      @index.each do |entry|
+        while latest.any? and latest[0] < entry.start
+          latest.shift
+        end
+
+        entry.overlap = latest.length
+        latest << entry.end
+      end
+    end
+
+    if @persistence_file
+      record_size = FixWidthTable.get_record_size(@index)
+
+      table = FixWidthTable.new @persistence_file, record_size, @range
+      @index.each do |entry| table.add entry, entry.start end
+      table.read
+
+      @index = table
+    end
+  end
+
+  def closest(pos)
+    upper = @index.size - 1
+    lower = 0
+
+    return -1 if upper < lower
+
+    while(upper >= lower) do
+      idx = lower + (upper - lower) / 2
+      comp = pos <=> @index[idx].start
+
+      if comp == 0
+        break 
+      elsif comp > 0
+        lower = idx + 1
+      else
+        upper = idx - 1
+      end
+    end
+
+    if @index[idx].start > pos
+      idx = idx - 1
+    end
+
+    idx
+  end
+
+  def get_range(pos)
+    if Range === pos
+      r_start = pos.begin
+      r_end   = pos.end
+    else
+      r_start = pos.to_i
+      r_end   = pos.to_i
+    end
+
+    if FixWidthTable === @index
+      idx = @index.closest(r_start)
+    else
+      idx = closest(r_start)
+    end
+
+    return [] if idx >= @index.size
+    return [] if idx <0 and r_start == r_end
+
+    idx = 0 if idx < 0
+
+    idx -= @index[idx].overlap if @index[idx].overlap
+
+    values = []
+    l = @index[idx]
+    while l.start <= r_end
+      values << l.value if l.end >= r_start 
+      idx += 1
+      break if idx >= @index.size
+      l = @index[idx]
+    end
+
+    values
+  end
+
+  def get_point(pos)
+    if Range === pos
+      r_start = pos.begin
+      r_end   = pos.end
+    else
+      r_start = pos.to_i
+      r_end   = pos.to_i
+    end
+
+    if FixWidthTable === @index
+      idx = @index.closest(r_start)
+    else
+      idx = closest(r_start)
+    end
+
+    return [] if idx >= @index.size
+    return [] if idx <0 and r_start == r_end
+
+    idx = 0 if idx < 0
+
+    idx += 1 unless @index[idx].start >= r_start
+
+    values = []
+    l = @index[idx]
+    while l.start <= r_end
+      values << l.value
+      idx += 1
+      break if idx >= @index.size
+      l = @index[idx]
+    end
+
+    values
+  end
+
+  def [](pos)
+    if range
+      get_range(pos)
+    else
+      get_point(pos)
+    end
+  end
+
+end

data/lib/rbbt/util/cmd.rb

@@ -3,7 +3,7 @@ require 'rbbt/util/log'
 require 'stringio'
 
 module CMD
-  class CMDError < RBBTError;end
+  class CMDError < RBBTError; end
 
   module SmartIO 
     def self.tie(io, pid = nil, cmd = "",  post = nil)
@@ -14,6 +14,7 @@ module CMD
         alias original_close close
         def close
           begin
+            self.original_read unless self.closed? or self.eof?
             Process.waitpid(@pid) if @pid
           rescue
           end

data/lib/rbbt/util/data_module.rb

@@ -1,5 +1,6 @@
 module DataModule
 
+  attr_accessor :sharedir, :rakefile, :pkg_module
   def self.extended(base)
     if defined? base::PKG and base::PKG
       base.pkg_module = base::PKG
@@ -8,41 +9,14 @@ module DataModule
     end
 
     base.sharedir = PKGData.get_caller_sharedir 
-  end
-
-  def pkg_module
-    @pkg_module
-  end
-
-  def pkg_module=(pkg_module)
-    @pkg_module = pkg_module
-  end
-
-  def sharedir
-    @sharedir
-  end
-
-  def sharedir=(sharedir)
-    @sharedir = sharedir
-  end
-
-  alias old_method_missing method_missing
-  def method_missing(name, *args, &block)
-    if args.any?
-      filename = File.join(self.to_s, args.first, name.to_s)
-    else
-      filename = File.join(self.to_s, name.to_s)
-    end
-
-    begin
-      pkg_module.add_datafiles filename => ['', self.to_s, sharedir]
-    rescue 
-      Log.debug $!.message
-      Log.debug $!.backtrace * "\n"
-      old_method_missing name, *args, &block
+    
+    Dir.glob(File.join(base.sharedir, 'install', base.to_s, '**','Rakefile')).each do |rakefile|
+      RakeHelper.files(rakefile).each do |file|
+        base.pkg_module.claim file, 
+          rakefile.sub(/^#{Regexp.quote File.join(base.sharedir)}\/?/,''), 
+          File.dirname(rakefile).sub(/^#{Regexp.quote File.join(base.sharedir, 'install')}\/?/,'')
+      end
     end
-
-    pkg_module.find_datafile filename
   end
 
   module WithKey
@@ -79,4 +53,21 @@ module DataModule
     o.key = key
     o
   end
+
+  alias old_method_missing method_missing
+  def method_missing(name, *args, &block)
+    begin
+      if args.any?
+        pkg_module.files[self.to_s][args.first][name] 
+      else
+        pkg_module.files[self.to_s][name] 
+      end
+    rescue
+      Log.debug $!.message
+      Log.debug $!.backtrace * "\n"
+      old_method_missing name, *args, &block
+    end
+  end
+
+
 end