rbbt-util 1.2.1 → 2.0.1

data/lib/rbbt-util.rb

@@ -4,6 +4,7 @@ require 'rbbt/util/cmd'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/filecache'
 require 'rbbt/util/tsv'
+require 'rbbt/util/persistence'
 require 'rbbt/util/bed'
 require 'rbbt/util/cachehelper'
 require 'rbbt/util/misc'
@@ -11,5 +12,5 @@ require 'rbbt/util/misc'
 FileCache.cachedir = Rbbt.cachedir
 Open.cachedir      = File.join(Rbbt.cachedir, 'open-remote/')
 TmpFile.tmpdir     = File.join(Rbbt.tmpdir)
-TSV.cachedir       = File.join(Rbbt.cachedir, 'tsv_cache')
+Persistence.cachedir       = File.join(Rbbt.cachedir, 'persistence')
 Bed.cachedir       = File.join(Rbbt.cachedir, 'bed_cache')

data/lib/rbbt/util/R.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/cmd'
+require 'rbbt/util/tsv'
 
 module R
 
@@ -18,7 +19,23 @@ module R
 
     Log.debug "R Script:\n#{ cmd }"
 
-    CMD.cmd('R --vanilla --slave', options.merge(:in => cmd))
+    CMD.cmd('R --vanilla --slave --quiet', options.merge(:in => cmd))
   end
 
 end
+
+class TSV
+  def R(script)
+    TmpFile.with_file do |f|
+      Open.write(f, self.to_s)
+      Log.debug(R.run(
+      <<-EOF
+data = rbbt.tsv('#{f}');
+#{script.strip}
+rbbt.tsv.write('#{f}', data);
+      EOF
+      ).read)
+      TSV.new(f, :type => :list)
+    end
+  end
+end

data/lib/rbbt/util/cmd.rb

@@ -82,12 +82,13 @@ module CMD
     sout, serr = IO.pipe, IO.pipe
 
     case 
-    when (IO === in_content and not StringIO === in_content)
+    when (false and (IO === in_content and not StringIO === in_content))
       sin = [in_content, nil]
-    else StringIO === in_content
+    else 
       sin = IO.pipe
     end
 
+
     pid = fork {
       begin
 
@@ -95,7 +96,6 @@ module CMD
         STDIN.reopen sin.first
         sin.first.close
 
-
         serr.first.close
         STDERR.reopen serr.last
         serr.last.close
@@ -115,16 +115,17 @@ module CMD
     sout.last.close
     serr.last.close
 
+
     Log.debug "CMD: [#{pid}] #{cmd}"
 
     case 
     when String === in_content
       sin.last.write in_content
       sin.last.close
-    when StringIO === in_content
+    when IO === in_content
       Thread.new do
-        while l = in_content.gets
-          sin.last.write l
+        while not in_content.eof?
+          sin.last.write in_content.gets
         end
         sin.last.close
       end

data/lib/rbbt/util/data_module.rb

@@ -1,24 +1,46 @@
 module DataModule
 
+  def self.rakefiles(sharedir, data_module)
+    Dir.glob(File.join(sharedir, 'install', data_module.to_s, '**','Rakefile')).collect{|f| File.expand_path f}
+  end
+
   attr_accessor :sharedir, :rakefile, :pkg_module
-  def self.extended(base)
-    if defined? base::PKG and base::PKG
-      base.pkg_module = base::PKG
+  def self.extended(data_module)
+    if defined? data_module::PKG and data_module::PKG
+      data_module.pkg_module = data_module::PKG
     else
-      base.pkg_module = Rbbt
+      data_module.pkg_module = Rbbt
     end
 
-    base.sharedir = PKGData.get_caller_sharedir 
+    data_module.sharedir = PKGData.get_caller_sharedir 
     
-    Dir.glob(File.join(base.sharedir, 'install', base.to_s, '**','Rakefile')).each do |rakefile|
+    install_dir = File.join(data_module.sharedir, 'install')
+    rake_sharedir  = File.join(data_module.sharedir, 'install')
+
+    rakefiles(data_module.sharedir, data_module).each do |rakefile|
+      rakefile_dir = File.dirname(rakefile)
+
       RakeHelper.files(rakefile).each do |file|
-        base.pkg_module.claim file, 
-          rakefile.sub(/^#{Regexp.quote File.join(base.sharedir)}\/?/,''), 
-          File.dirname(rakefile).sub(/^#{Regexp.quote File.join(base.sharedir, 'install')}\/?/,'')
+        file_path = Misc.path_relative_to(File.join(File.dirname(rakefile), file), rakefile_dir)
+        get       = :Rakefile
+        subdir    = Misc.path_relative_to(File.dirname(rakefile), install_dir)
+        namespace = data_module.to_s.gsub('/', '::')
+
+        data_module.pkg_module.claim file_path, get, subdir, namespace, rake_sharedir
       end
     end
   end
 
+  def files
+    DataModule.rakefiles(sharedir, self).collect do |rakefile|
+      RakeHelper.files(rakefile).collect
+    end.flatten
+  end
+
+  def datadir
+    File.join(pkg_module.datadir, self.to_s)
+  end
+
   module WithKey
     def klass=(klass)
       @klass = klass
@@ -68,6 +90,4 @@ module DataModule
       old_method_missing name, *args, &block
     end
   end
-
-
 end

data/lib/rbbt/util/fix_width_table.rb

@@ -0,0 +1,209 @@
+
+class FixWidthTable
+
+  attr_accessor :filename, :file, :value_size, :record_size, :range, :size
+  def initialize(filename, value_size = nil, range = nil, update = false)
+    @filename = filename
+
+    if update or not File.exists? filename
+      Log.debug "Writing FixWidthTable at #{ @filename.inspect }"
+      FileUtils.rm @filename if File.exists? @filename
+      @value_size = value_size
+      @range = range
+      @record_size = @value_size + (@range ? 12 : 4)
+      @file = File.open(@filename, 'wb')
+      @file.write [value_size].pack("L")
+      @file.write [@range ? 1 : 0 ].pack("C")
+      @size = 0
+    else
+      Log.debug "Reading FixWidthTable at #{ @filename.inspect }"
+      @file = File.open(@filename, 'r')
+      @value_size = @file.read(4).unpack("L").first
+      @range       = @file.read(1).unpack("C").first == 1
+      @record_size = @value_size + (@range ? 12 : 4)
+      @size        = (File.size(@filename) - 5) / (@record_size)
+    end
+  end
+
+  def format(pos, value)
+    padding = value_size - value.length
+    if range
+      (pos  + [value + "\0" * padding]).pack("llla#{value_size}")
+    else
+      [pos, value + "\0" * padding].pack("la#{value_size}")
+    end
+  end
+
+  def unformat(format)
+    if range
+      pos_start, pos_end, pos_overlap, value = format.unpack("llla#{value_size}")
+      [[pos_start, pos_end, pos_overlap], value.strip]
+    else
+      pos, value = format.unpack("la#{value_size}")
+      [pos, value.strip]
+    end
+  end
+
+  def add(pos, value)
+    format = format(pos, value)
+    @file.write format
+    @size += 1
+  end
+  alias << add
+
+  def pos(index)
+    return nil if index < 0 or index >= size
+    @file.seek(5 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def pos_end(index)
+    return nil if index < 0 or index >= size
+    @file.seek(9 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def overlap(index)
+    return nil if index < 0 or index >= size
+    @file.seek(13 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def value(index)
+    return nil if index < 0 or index >= size
+    @file.seek((range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
+    @file.read(value_size).unpack("a#{value_size}").first.strip
+  end
+
+  def read
+    @file.close unless @file.closed?
+    @file = File.open(@filename, 'r')
+  end
+
+  def close
+    @file.close
+  end
+
+  #{{{ Adding data
+
+  def add_point(data)
+    data.sort_by{|value, pos| pos}.each do |value, pos|
+      add pos, value
+    end
+  end
+
+  def add_range(data)
+    latest = []
+    data.sort_by{|value, pos| pos[0]}.each do |value, pos|
+      while latest.any? and latest[0] < pos[0]
+        latest.shift
+      end
+
+      overlap = latest.length
+
+      add pos + [overlap], value
+      latest << pos[1]
+    end
+  end
+
+  #{{{ Searching
+
+  def closest(pos)
+    upper = size - 1
+    lower = 0
+
+    return -1 if upper < lower
+
+    while(upper >= lower) do
+      idx = lower + (upper - lower) / 2
+      comp = pos <=> pos(idx)
+
+      if comp == 0
+        break 
+      elsif comp > 0
+        lower = idx + 1
+      else
+        upper = idx - 1
+      end
+    end
+
+    if pos(idx) > pos
+      idx = idx - 1
+    end
+
+    idx
+  end
+
+  def get_range(pos)
+    if Range === pos
+      r_start = pos.begin
+      r_end   = pos.end
+    else
+      r_start = pos.to_i
+      r_end   = pos.to_i
+    end
+
+    idx = closest(r_start)
+
+    return [] if idx >= size
+    return [] if idx <0 and r_start == r_end
+
+    idx = 0 if idx < 0
+
+    idx -= overlap(idx) unless overlap(idx).nil?
+
+    values = []
+    l_start = pos(idx)
+    l_end   = pos_end(idx)
+    while l_start <= r_end
+      values << value(idx) if l_end >= r_start 
+      idx += 1
+      break if idx >= size
+      l_start = pos(idx)
+      l_end   = pos_end(idx)
+    end
+
+    values
+  end
+
+  def get_point(pos)
+    if Range === pos
+      r_start = pos.begin
+      r_end   = pos.end
+    else
+      r_start = pos.to_i
+      r_end   = pos.to_i
+    end
+
+    idx = closest(r_start)
+
+    return [] if idx >= size
+    return [] if idx <0 and r_start == r_end
+
+    idx = 0 if idx < 0
+
+    idx += 1 unless pos(idx) >= r_start
+
+    values = []
+    l_start = pos(idx)
+    l_end   = pos_end(idx)
+    while l_start <= r_end
+      values << value(idx)
+      idx += 1
+      break if idx >= size
+      l_start = pos(idx)
+      l_end   = pos_end(idx)
+    end
+
+    values
+  end
+
+  def [](pos)
+    if range
+      get_range(pos)
+    else
+      get_point(pos)
+    end
+  end
+
+end

data/lib/rbbt/util/log.rb

@@ -14,6 +14,7 @@ module Log
   end
 
   def self.log(message, severity = MEDIUM)
+    STDERR.puts caller * "\n" if @@severity == -1 and not message.empty?
     STDERR.puts "#{Time.now}[#{severity.to_s}]: " +  message if severity >= @@severity
   end
 
@@ -50,6 +51,15 @@ module Log
 end
 
 def ddd(message, file = $stdout)
-  Log.debug "DEVEL => " << caller.first
-  Log.debug "DEVEL: " << message.inspect
+  Log.debug "DEVEL: " << caller.first
+  Log.debug ""
+  Log.debug "=> " << message.inspect
+  Log.debug ""
+end
+
+def ppp(message)
+  puts "PRINT: " << caller.first
+  puts ""
+  puts "=> " << message.inspect
+  puts ""
 end

data/lib/rbbt/util/misc.rb

@@ -16,6 +16,29 @@ end
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  def self.string2const(string)
+    return nil if string.nil?
+    mod = Kernel
+
+    string.to_s.split('::').each do |str|
+      mod = mod.const_get str
+    end
+
+    mod
+  end
+
+  def self.path_relative_to(path, subdir)
+    File.expand_path(path).sub(/^#{Regexp.quote File.expand_path(subdir)}\/?/,'')
+  end
+
+  def self.in_directory?(file, directory)
+    if file.to_s =~ /^#{Regexp.quote File.expand_path(directory)}/
+      true
+    else
+      false
+    end
+  end
+
   def self.this_dir
     File.expand_path(File.dirname(caller[0]))
   end
@@ -75,7 +98,9 @@ module Misc
       raise "Format of '#{options.inspect}' not understood"
     end
     defaults.each do |key, value|
-      new_options[key] = value if new_options[key].nil?
+      next unless new_options[key].nil?
+
+      new_options[key] = value 
     end
     new_options
   end
@@ -88,6 +113,14 @@ module Misc
     end
   end
 
+  def self.hash2string(hash)
+    hash.collect{|k,v| 
+      next unless %w(Symbol String Float Fixnum Integer TrueClass FalseClass Module Class Object).include? v.class.to_s
+      [ Symbol === k ? ":" << k.to_s : k,
+        Symbol === v ? ":" << v.to_s : v] * "="
+    }.compact * "#"
+  end
+
   def self.string2hash(string)
 
     options = {}
@@ -104,7 +137,27 @@ module Misc
       if value == true
         options[option] = option.to_s.chars.first != '!' 
       else
-        options[option] = begin eval(value) rescue value end
+        options[option] = Thread.start do
+          $SAFE = 0;
+          case 
+          when value =~ /^(?:true|T)$/i
+            true
+          when value =~ /^(?:false|F)$/i
+            false
+          when (String === value and value =~ /^\/(.*)\/$/)
+            Regexp.new /#{$1}/
+          else
+            begin
+              Kernel.const_get value
+            rescue
+              begin  
+                eval(value) 
+              rescue Exception
+                value 
+              end
+            end
+          end
+        end.value
       end
     end
 
@@ -135,7 +188,7 @@ module Misc
     fields.each_with_index{|f,i| return i if f == field}
     field_re = Regexp.new /#{field}/i
     fields.each_with_index{|f,i| return i if f =~ field_re}
-    raise FieldNotFoundError, "Field '#{ field }' was not found" unless quiet
+    raise FieldNotFoundError, "Field #{ field.inspect } was not found" unless quiet
   end
 
   def self.first(list)
@@ -143,12 +196,17 @@ module Misc
     return list.first
   end
 
+  def self.chunk(text, split)
+    text.split(split)[1..-1]
+  end
+
 end
 
 module PDF2Text
   def self.pdf2text(filename)
     require 'rbbt/util/cmd'
     require 'rbbt/util/tmpfile'
+    require 'rbbt/util/open'
     TmpFile.with_file(Open.read(filename)) do |pdf|
       CMD.cmd("pdftotext #{pdf} -", :pipe => false, :stderr => true)
     end
@@ -191,22 +249,43 @@ class NamedArray < Array
     zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields 
     zipped 
   end
+
+  def detach(file)
+    file_fields = file.fields.collect{|field| field.fullname}
+    detached_fields = []
+    self.fields.each_with_index{|field,i| detached_fields << i if file_fields.include? field.fullname}
+    fields = self.fields.values_at *detached_fields
+    values = self.values_at *detached_fields
+    values = NamedArray.name(values, fields)
+    values.zip_fields
+  end
 end
 
-def benchmark
+def benchmark(bench = true)
   require 'benchmark'
-  puts(Benchmark.measure do
+  if bench
+    res = nil
+    puts(Benchmark.measure do
+      res = yield
+    end)
+    res
+  else
     yield
-  end)
+  end
 end
 
-def profile 
+def profile(prof = true)
   require 'ruby-prof'
-  RubyProf.start
-  yield
-  result = RubyProf.stop
+  if prof
+    RubyProf.start
+    res = yield
+    result = RubyProf.stop
 
   # Print a flat profile to text
-  printer = RubyProf::FlatPrinter.new(result)
-  printer.print(STDOUT, 0)
+    printer = RubyProf::FlatPrinter.new(result)
+    printer.print(STDOUT, 0)
+    res
+  else
+    yield
+  end
 end