rbbt-util 2.1.0 → 3.0.2

data/bin/rbbt_query.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/sources/organism'
+
+options = SOPT.get("-i--identifiers*:-f--format*:-o--organism*:-p--persistence:-l--log*:-r--report:-h--hash")
+
+file   = ARGV[0]
+
+if not File.exists? file
+  base, path = file.match(/(.*)?\.(.*)/).values_at 1, 2
+  require 'rbbt/sources/' << base.to_s.downcase
+  klass = Misc.string2const base
+  file = klass[path].find
+end
+
+entities = ARGV[1].nil? ? '-' : ARGV[1].dup
+persistence = options[:persistence]
+log = (options[:log] || 4).to_i
+Log.severity = log
+organism = options[:organism] || "Hsa"
+identifiers = options[:identifiers] || Organism.identifiers(organism)
+format = options[:format] 
+report = options[:report]
+
+if format.to_s == "key"
+  File.open(file) do |f|
+    format = TSV.parse_header(f).first
+  end
+end
+
+if format.nil?
+  f, count = TSV.new(identifiers).guess_field(entities)
+  format   = f if count > 0
+end
+
+data = if entities == '-'
+  TSV.new(STDIN)
+  #entities = TSV.new(STDIN.read.split(/\n|\||\t/)
+else
+  entities = [entities]
+  data = TSV.new(entities) 
+  data.type = :double
+  data.identifiers = identifiers
+  data.key_field = format.dup unless FalseClass === format or format.nil?
+  data.fields ||= []
+  data
+end
+
+data.type = :double
+data.identifiers = identifiers
+
+data.attach TSV.new file, :persistence => persistence
+
+if report
+  data.each do |entity,values|
+    puts "== Entity: #{ entity }"
+    puts values.report
+  end
+else
+  puts data
+end

data/lib/rbbt-util.rb

@@ -9,8 +9,8 @@ require 'rbbt/util/bed'
 require 'rbbt/util/cachehelper'
 require 'rbbt/util/misc'
 
-FileCache.cachedir = Rbbt.cachedir
-Open.cachedir      = File.join(Rbbt.cachedir, 'open-remote/')
-TmpFile.tmpdir     = File.join(Rbbt.tmpdir)
-Persistence.cachedir       = File.join(Rbbt.cachedir, 'persistence')
-Bed.cachedir       = File.join(Rbbt.cachedir, 'bed_cache')
+FileCache.cachedir   = Rbbt.var.cache.filecache.find :user
+Open.cachedir        = Rbbt.var.cache["open-remote"].find :user
+TmpFile.tmpdir       = Rbbt.tmp.find :user
+Persistence.cachedir = Rbbt.var.cache.persistence.find :user
+Bed.cachedir         = Rbbt.var.cache["bed-persistence"].find :user

data/lib/rbbt.rb

@@ -1,16 +1,7 @@
-require 'rbbt/util/pkg_config'
-require 'rbbt/util/pkg_data'
-require 'rbbt/util/pkg_software'
-require 'rbbt/util/open'
-require 'rbbt/util/tmpfile'
-require 'rbbt/util/filecache'
+require 'rbbt/util/resource'
 
 module Rbbt
-  extend PKGConfig
-  extend PKGData
-  extend PKGSoftware
-
-  self.load_cfg(%w(tmpdir cachedir datadir))
+  extend Resource
 end
 
 

data/lib/rbbt/util/cmd.rb

@@ -122,7 +122,7 @@ module CMD
     when String === in_content
       sin.last.write in_content
       sin.last.close
-    when IO === in_content
+    when in_content.respond_to?(:gets)
       Thread.new do
         while not in_content.eof?
           sin.last.write in_content.gets

data/lib/rbbt/util/fix_width_table.rb

@@ -5,13 +5,19 @@ class FixWidthTable
   def initialize(filename, value_size = nil, range = nil, update = false)
     @filename = filename
 
-    if update or not File.exists? filename
+    if update or %(memmory stringio).include?(filename.to_s.downcase) or not File.exists? filename
       Log.debug "Writing FixWidthTable at #{ @filename.inspect }"
-      FileUtils.rm @filename if File.exists? @filename
       @value_size = value_size
       @range = range
       @record_size = @value_size + (@range ? 12 : 4)
-      @file = File.open(@filename, 'wb')
+
+      if %(memory stringio).include? filename.to_s.downcase
+        @file = StringIO.new
+      else
+        FileUtils.rm @filename if File.exists? @filename
+        @file = File.open(@filename, 'wb')
+      end
+
       @file.write [value_size].pack("L")
       @file.write [@range ? 1 : 0 ].pack("C")
       @size = 0

data/lib/rbbt/util/log.rb

@@ -1,9 +1,12 @@
 module Log
 
-  DEBUG  = 0
-  LOW    = 1
-  MEDIUM = 2
-  HIGH   = 3
+  DEBUG    = 0
+  LOW      = 1
+  MEDIUM   = 2
+  HIGH     = 3
+  INFO     = 4
+  WARN     = 5
+  ERROR    = 6
 
   def self.severity=(severity)
     @@severity = severity
@@ -13,9 +16,13 @@ module Log
     @@severity
   end
 
+  SEVERITY_COLOR = ["0;37m", "32m", "33m", "31m", "1;0m" ].collect{|e| "\033[#{e}"}
+
   def self.log(message, severity = MEDIUM)
-    STDERR.puts caller * "\n" if @@severity == -1 and not message.empty?
-    STDERR.puts "#{Time.now}[#{severity.to_s}]: " +  message if severity >= @@severity
+    severity_color = SEVERITY_COLOR[severity]
+    STDERR.puts caller.select{|l| l =~ /rbbt/} * "\n" if @@severity == -1 and not message.empty?
+    #STDERR.puts "#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}[#{severity.to_s}]: " +  message if severity >= @@severity
+    STDERR.puts "\033[0;37m#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}#{severity_color}[#{severity.to_s}]\033[0m: " +  message if severity >= @@severity
   end
 
   def self.debug(message)
@@ -34,6 +41,15 @@ module Log
     log(message, HIGH)
   end
 
+  def self.warn(message)
+    log(message, WARN)
+  end
+
+  def self.error(message)
+    log(message, ERROR)
+  end
+
+
   case ENV['RBBT_LOG']
   when 'DEBUG' 
     @@severity = DEBUG
@@ -44,7 +60,7 @@ module Log
   when 'HIGH' 
     @@severity = HIGH
   when nil
-    @@severity = HIGH
+    @@severity = INFO
   else
     @@severity = ENV['RBBT_LOG'].to_i
   end

data/lib/rbbt/util/misc.rb

@@ -1,11 +1,12 @@
 require 'iconv'
+require 'digest/md5'
 
 class RBBTError < StandardError
   attr_accessor :info
 
   alias old_to_s to_s
   def to_s
-    str = old_to_s
+    str = old_to_s.dup
     if info
       str << "\n" << "Additional Info:\n---\n" << info << "---"
     end
@@ -32,13 +33,26 @@ module Misc
   end
 
   def self.in_directory?(file, directory)
-    if file.to_s =~ /^#{Regexp.quote File.expand_path(directory)}/
+    if File.expand_path(file) =~ /^#{Regexp.quote File.expand_path(directory)}/
       true
     else
       false
     end
   end
 
+  def self.find_files_back_to(path, target, subdir)
+    return [] if path.nil?
+    files = []
+    while in_directory?(path, subdir)
+      path = path.dirname
+      if path[target].exists?
+        files << path[target]
+      end
+    end
+
+    files
+  end
+
   def self.this_dir
     File.expand_path(File.dirname(caller[0]))
   end
@@ -46,11 +60,11 @@ module Misc
   def self.env_add(var, value, sep = ":", prepend = true)
     ENV[var] ||= ""
     return if ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
-    if prepend
-      ENV[var] = value + sep + ENV[var]
-    else
-      ENV[var] += sep + ENV[var]
-    end
+      if prepend
+        ENV[var] = value + sep + ENV[var]
+      else
+        ENV[var] += sep + ENV[var]
+      end
   end
 
   def self.count(list)
@@ -114,13 +128,26 @@ module Misc
   end
 
   def self.hash2string(hash)
-    hash.collect{|k,v| 
+    hash.sort_by{|k,v| k.to_s}.collect{|k,v| 
       next unless %w(Symbol String Float Fixnum Integer TrueClass FalseClass Module Class Object).include? v.class.to_s
       [ Symbol === k ? ":" << k.to_s : k,
         Symbol === v ? ":" << v.to_s : v] * "="
     }.compact * "#"
   end
 
+  def self.hash2md5(hash)
+    o = {}
+    hash.each do |k,v|
+      if v.inspect =~ /:0x0/
+        o[k] = v.inspect.sub(/:0x[a-f0-9]+@/,'')
+      else
+        o[k] = v
+      end
+    end
+
+    Digest::MD5.hexdigest(o.inspect)
+  end
+
   def self.string2hash(string)
 
     options = {}
@@ -200,6 +227,62 @@ module Misc
     text.split(split)[1..-1]
   end
 
+  def self.insist(times = 3)
+    try = 0
+    begin
+      yield
+    rescue
+      try += 1
+      retry if try < times
+    end
+  end
+
+  def self.try3times(&block)
+    insist(3, &block)
+  end
+
+
+  # Divides the array into +num+ chunks of the same size by placing one
+  # element in each chunk iteratively.
+  def self.divide(array, num)
+    chunks = [[]] * num
+    array.each_with_index{|e, i|
+      c = i % num
+      chunks[c] << e
+    }
+    chunks
+  end
+
+  IUPAC2BASE = {
+    "A" => ["A"],
+    "C" => ["C"],
+    "G" => ["G"],
+    "T" => ["T"],
+    "U" => ["U"],
+    "R" => "A or G".split(" or "),
+    "Y" => "C or T".split(" or "),
+    "S" => "G or C".split(" or "),
+    "W" => "A or T".split(" or "),
+    "K" => "G or T".split(" or "),
+    "M" => "A or C".split(" or "),
+    "B" => "C or G or T".split(" or "),
+    "D" => "A or G or T".split(" or "),
+    "H" => "A or C or T".split(" or "),
+    "V" => "A or C or G".split(" or "),
+    "N" => %w(A C T G),
+  }
+
+  BASE2COMPLEMENT = {
+    "A" => "T",
+    "C" => "G",
+    "G" => "C",
+    "T" => "A",
+    "U" => "A",
+  }
+
+  def self.IUPAC_to_base(iupac)
+    IUPAC2BASE[iupac]
+  end
 end
 
 module PDF2Text
@@ -222,10 +305,27 @@ class NamedArray < Array
     a
   end
 
+  def merge(array)
+    double = Array === array.first 
+    new = self.dup
+    (0..length - 1).each do |i|
+      if double
+        new[i] = new[i] + array[i]
+      else
+        new[i] << array[i]
+      end
+    end
+    new
+  end
+
   def positions(fields)
-    fields.collect{|field|
-      Misc.field_position(@fields, field)
-    }
+    if Array ==  fields
+      fields.collect{|field|
+        Misc.field_position(@fields, field)
+      }
+    else
+      Misc.field_position(@fields, fields)
+    end
   end
 
   alias original_get_brackets []
@@ -234,9 +334,9 @@ class NamedArray < Array
   end
 
   alias original_set_brackets []=
-  def []=(key,value)
-    original_set_brackets(Misc.field_position(fields, key), value)
-  end
+    def []=(key,value)
+      original_set_brackets(Misc.field_position(fields, key), value)
+    end
 
   alias original_values_at values_at
   def values_at(*keys)
@@ -259,6 +359,12 @@ class NamedArray < Array
     values = NamedArray.name(values, fields)
     values.zip_fields
   end
+
+  def report
+    fields.zip(self).collect do |field,value|
+      "* #{ field }: #{ Array === value ? value * "|" : value }"
+    end * "\n"
+  end
 end
 
 def benchmark(bench = true)
@@ -281,7 +387,7 @@ def profile(prof = true)
     res = yield
     result = RubyProf.stop
 
-  # Print a flat profile to text
+    # Print a flat profile to text
     printer = RubyProf::FlatPrinter.new(result)
     printer.print(STDOUT, 0)
     res

data/lib/rbbt/util/open.rb

@@ -203,12 +203,22 @@ module Open
   def self.write(file, content)
     FileUtils.mkdir_p File.dirname(file)
     if String === content
-      File.open(file, 'w') do |f| f.write content end
+      File.open(file, 'w') do |f|
+        f.flock(File::LOCK_EX)
+        f.write content 
+        f.flock(File::LOCK_UN)
+      end
     else
-      File.open(file, 'w') do |f| 
-        while l = content.gets
-          f.write l
+      begin
+        File.open(file, 'w') do |f| 
+          f.flock(File::LOCK_EX)
+          while l = content.gets
+            f.write l
+          end
+          f.flock(File::LOCK_UN)
         end
+      rescue
+        FileUtils.rm file if File.exists? file
       end
       content.close
     end

data/lib/rbbt/util/persistence.rb

@@ -1,6 +1,7 @@
 require 'rbbt/util/tsv'
 require 'rbbt/util/misc'
 require 'rbbt/util/open'
+require 'digest/md5'
 require 'yaml'
 
 module Persistence
@@ -20,17 +21,12 @@ module Persistence
   end
  
   def self.get_persistence_file(file, prefix, options = {})
+    persistence_dir = Misc.process_options options, :persistence_dir
+    persistence_dir ||= CACHEDIR
     name = prefix.to_s << ":" << file.to_s << ":"
-    o = {}
-    options.each do |k,v|
-      if v.inspect =~ /:0x0/
-        o[k] = v.inspect.sub(/:0x[a-f0-9]+@/,'')
-      else
-        o[k] = v
-      end
-    end
 
-    File.join(CACHEDIR, name.to_s.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, o].inspect))
+    options_md5 = Misc.hash2md5 options
+    File.join(persistence_dir, name.to_s.gsub(/\s/,'_').gsub(/\//,'>') + options_md5)
   end
 
   def self.get_filename(file)
@@ -132,16 +128,36 @@ module Persistence
     end
   end
 
-  def self.tsv_serializer(data)
-    case
-    when (not Object::TSV === data)
-      :marshal
-    when data.type == :double
-      :double
-    when data.type == :single
-      :single
+  def self.tsv_serializer(data, extra = nil)
+    if Object::TSV === data
+      return :integer if (data.cast == "to_i" or data.cast == :to_i) and data.type == :single
+      return :integer_array if (data.cast == "to_i" or data.cast == :to_i) and (data.type == :list or data.type == :flat)
+
+      case
+      when data.type == :double
+        :double
+      when data.type == :list
+        :list
+      when data.type == :single
+        :single
+      else
+        :marshal
+      end
     else
-      :list
+      return :marshal if extra.nil?
+      return :integer if (extra[:cast] == "to_i" or extra[:cast] == :to_i) and extra[:type] == :single
+      return :integer_array if (extra[:cast] == "to_i" or extra[:cast] == :to_i) and (extra[:type] == :list or extra[:type] == :flat)
+
+      case
+      when extra[:type] == :double
+        :double
+      when extra[:type] == :list
+        :list
+      when extra[:type] == :single
+        :single
+      else
+        :marshal
+      end
     end
   end
 
@@ -208,7 +224,8 @@ module Persistence
     if persistence_update or not File.exists? persistence_file
       Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
       res, extra = yield file, options, filename, persistence_file
-      serializer = tsv_serializer res
+
+      serializer = tsv_serializer res, extra
 
       per = Persistence::TSV.get persistence_file, true, serializer
 
@@ -258,7 +275,6 @@ module Persistence
         FileUtils.rm persistence_file
       end
 
-
       max_length = res.collect{|k,v| k.length}.max
 
       if range
@@ -266,7 +282,7 @@ module Persistence
           fwt = FixWidthTable.new persistence_file, max_length, true
           fwt.add_range res
         rescue
-          FileUtils.rm persistence_file
+          FileUtils.rm persistence_file if File.exists? persistence_file
           raise $!
         end
       else
@@ -298,6 +314,19 @@ module Persistence
 
     filename = get_filename(file)
 
+    if persistence == :no_create
+      o = options.dup
+      options = 
+        Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
+      persistence_update, persistence_dir, persistence_file, filename =
+        Misc.process_options options, :persistence_update, :persistence_dir, :persistence_file, :filename
+
+      filename         ||= get_filename(file)
+      persistence_file ||= get_persistence_file(filename, prefix, options.merge(:persistence_dir => persistence_dir))
+
+      persistence = false if not File.exists? persistence_file
+    end
+
     if not persistence
       Log.low "Non Persistent Loading for #{filename}. Prefix: #{prefix}"
       yield file, options, filename
@@ -322,4 +351,6 @@ module Persistence
       end
     end
   end
+
+
 end