rbbt-util 5.2.4 → 5.3.0

data/lib/rbbt/util/colorize.rb

@@ -0,0 +1,57 @@
+require 'rbbt/util/color'
+
+module Colorize
+  def self.from_name(color)
+    return color if color =~ /^#?[0-9A-F]+$/i
+    case color.to_s
+    when "white"
+      '#000'
+    when "black"
+      '#fff'
+    end
+  end
+
+  def self.continuous(array, start = :white, eend = :black) 
+    start_color = Color.new from_name(start)
+    end_color = Color.new from_name(eend)
+
+    array = array.collect{|v| v.to_f}
+    max = array.max
+    min = array.min
+    range = max - min
+    array.collect do |v|
+      start_color.blend end_color, (v - min) / range
+    end
+  end
+
+  def self.distinct(array)
+    colors = Rbbt.share.color["diverging_colors.hex"].list.collect{|c| Color.new c}
+
+    num = array.uniq.length
+    times = num / 12
+
+    all_colors = colors.dup
+    times.times do
+      all_colors.concat  colors.collect{|n| n.darken(0.2) }
+    end
+
+    value_color = Hash[*array.uniq.zip(all_colors).flatten]
+
+    value_color.values_at *array
+  end
+
+  def self.tsv(tsv)
+    values = tsv.values.flatten
+    if Fixnum === values.first or (values.first.to_f != 0 and values[0] != "0")
+      value_colors = Misc.process_to_hash(values){continuous(values)}
+    else
+      value_colors = Misc.process_to_hash(values){distinct(values)}
+    end
+
+    if tsv.type == :single
+      Hash[*tsv.keys.zip(value_colors.values_at(*values)).flatten]
+    else
+      Hash[*tsv.keys.zip(values.collect{|vs| value_colors.values_at(*vs)}).flatten]
+    end
+  end
+end

data/lib/rbbt/util/misc.rb

@@ -22,6 +22,7 @@ module Misc
   def self.correct_icgc_mutation(pos, ref, mut_str)
     mut = mut_str
     mut = '-' * (mut_str.length - 1) if mut =~/^-[ACGT]/
+    mut = "+" << mut if ref == '-'
     [pos, [mut]]
   end
 
@@ -143,9 +144,13 @@ module Misc
   end
 
   def self.sample(ary, size, replacement = false)
-    total = ary.length
-    p = random_sample_in_range(total, size)
-    ary.values_at *p
+    if ary.respond_to? :sample
+      ary.sample size
+    else
+      total = ary.length
+      p = random_sample_in_range(total, size)
+      ary.values_at *p
+    end
   end
 
   Log2Multiplier = 1.0 / Math.log(2.0)
@@ -160,13 +165,14 @@ module Misc
 
     dup_array = options.delete :dup_array
 
-    if  Entity.respond_to?(:formats) and Entity.formats.include? field
+    if Annotated === field or Entity.respond_to?(:formats) and Entity.formats.include? field
       params = options.dup
 
       params[:format] ||= params.delete "format"
       params.merge!(:format => field) unless params.include?(:format) and not ((f = params[:format]).nil? or (String === f and f.empty?))
 
-      entity = Entity.formats[field].setup(
+      mod = Entity === field ? field : Entity.formats[field]
+      entity = mod.setup(
         ((entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) ),
         params
       ) 
@@ -414,15 +420,6 @@ end
     for spos in 0..cols-1 do a[spos, 0] = spos * init_gap end
     for rpos in 0..rows-1 do a[0, rpos] = rpos * init_gap end
 
-    #for spos in 1..cols-1 do
-    #  for rpos in 1..rows-1 do
-    #    match = a[spos-1,rpos-1] + (sequence[spos-1] != reference[rpos-1] ? diff : same)
-    #    skip_sequence = a[spos-1,rpos] + gap
-    #    skip_reference = a[spos,rpos-1] + gap
-    #    a[spos,rpos] = [match, skip_sequence, skip_reference].max
-    #  end
-    #end
-
     spos = 1
     while spos < cols do
       rpos = 1
@@ -760,7 +757,6 @@ end
     html
   end
 
-
   def self.path_relative_to(basedir, path)
     path = File.expand_path(path)
     basedir = File.expand_path(basedir)
@@ -775,6 +771,10 @@ end
     end
   end
 
+  def self.hostname
+    @hostanem ||= `hostname`.strip
+  end
+
   def self.lock(file, *args)
     return yield file, *args if file.nil?
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
@@ -785,7 +785,7 @@ end
 
     begin
       if File.exists? lockfile and
-        `hostname`.strip == (info = YAML.load_file(lockfile))["host"] and 
+        Misc.hostname == (info = YAML.load_file(lockfile))["host"] and 
         info["pid"] and not Misc.pid_exists?(info["pid"])
 
         Log.info("Removing lockfile: #{lockfile}. This pid #{Process.pid}. Content: #{info.inspect}")
@@ -803,6 +803,39 @@ end
     res
   end
 
+  LOCK_REPO_SERIALIZER=Marshal
+
+  def self.lock_in_repo(repo, key, *args)
+    return yield file, *args if repo.nil? or key.nil?
+
+    lock_key = "lock-" << key
+
+    begin
+      if repo[lock_key] and
+        Misc.hostname == (info = LOCK_REPO_SERIALIZER.load(repo[lock_key]))["host"] and 
+        info["pid"] and not Misc.pid_exists?(info["pid"])
+
+        Log.info("Removing lockfile: #{lock_key}. This pid #{Process.pid}. Content: #{info.inspect}")
+        repo.out lock_key 
+      end
+    rescue
+      Log.warn("Error checking lockfile #{lock_key}: #{$!.message}. Removing. Content: #{begin repo[lock_key] rescue "Could not open file" end}")
+      repo.out lock_key if repo.include? lock_key
+    end
+
+    while repo[lock_key]
+      sleep 1
+    end
+    
+    repo[lock_key] = LOCK_REPO_SERIALIZER.dump({:hostname => Misc.hostname, :pid => Process.pid})
+
+    res = yield lock_key, *args
+
+    repo.delete lock_key
+
+    res
+  end
+
   def self.common_path(dir, file)
     file = File.expand_path file
     dir = File.expand_path dir
@@ -913,6 +946,10 @@ end
     keys = keys.clean_annotations if keys.respond_to? :clean_annotations
     keys = keys.sort_by{|k| k.to_s}
 
+    if hash.respond_to? :unnamed
+      unnamed = hash.unnamed
+      hash.unnamed = true 
+    end
     keys.each do |k|
       next if k == :monitor or k == "monitor" or k == :in_situ_persistence or k == "in_situ_persistence"
       v = hash[k]
@@ -926,7 +963,7 @@ end
       when Symbol === v
         str << k.to_s << "=>" << v.to_s
       when (String === v and v.length > HASH2MD5_MAX_STRING_LENGTH)
-        str << k.to_s << "=>" << v[0..HASH2MD5_MAX_STRING_LENGTH]
+        str << k.to_s << "=>" << v[0..HASH2MD5_MAX_STRING_LENGTH] << "; #{ v.length }"
       when String === v
         str << k.to_s << "=>" << v
       when (Array === v and v.length > HASH2MD5_MAX_ARRAY_LENGTH)
@@ -947,6 +984,7 @@ end
 
       str << "_" << hash2md5(v.info) if Annotated === v
     end
+    hash.unnamed = unnamed if hash.respond_to? :unnamed
 
     if str.empty?
       ""
@@ -1044,8 +1082,8 @@ end
     return field if Integer === field or Range === field
     raise FieldNotFoundError, "Field information missing" if fields.nil? && ! quiet
     fields.each_with_index{|f,i| return i if f == field}
-    field_re = Regexp.new /#{field}/i
-      fields.each_with_index{|f,i| return i if f =~ field_re}
+    field_re = Regexp.new /^#{field}$/i
+    fields.each_with_index{|f,i| return i if f =~ field_re}
     raise FieldNotFoundError, "Field #{ field.inspect } was not found" unless quiet
   end
 

data/lib/rbbt/util/named_array.rb

@@ -1,10 +1,10 @@
-require 'rbbt/util/chain_methods'
+#require 'rbbt/util/chain_methods'
 require 'rbbt/util/misc'
 
 module NamedArray
-  extend ChainMethods
+  #extend ChainMethods
+  #self.chain_prefix = :named_array
 
-  self.chain_prefix = :named_array
   attr_accessor :fields
   attr_accessor :key
   attr_accessor :entity_options
@@ -36,7 +36,7 @@ module NamedArray
       if entity_templates.include? field
         entity
       else
-        template = Misc.prepare_entity("TEMPLATE", field, options)
+        template = Misc.prepare_entity("ENTITY_TEMPLATE", field, options)
         if Annotated === template
           entity_templates[field] = template
           entity = template.annotate(entity.frozen? ? entity.dup : entity)
@@ -73,9 +73,20 @@ module NamedArray
     end
   end
 
-  def named_array_get_brackets(key)
+  #def named_array_get_brackets(key)
+  #  pos = Misc.field_position(fields, key)
+  #  elem = named_array_clean_get_brackets(pos)
+
+  #  return elem if @fields.nil? or @fields.empty?
+
+  #  field = NamedArray === @fields ? @fields.named_array_clean_get_brackets(pos) : @fields[pos]
+  #  elem = prepare_entity(elem, field, entity_options)
+  #  elem
+  #end
+
+  def [](key, clean = false)
     pos = Misc.field_position(fields, key)
-    elem = named_array_clean_get_brackets(pos)
+    elem = super(pos)
 
     return elem if @fields.nil? or @fields.empty?
 
@@ -84,7 +95,19 @@ module NamedArray
     elem
   end
 
-  def named_array_each(&block)
+  #def named_array_each(&block)
+  #  if defined?(Entity) and not @fields.nil? and not @fields.empty?
+  #    @fields.zip(self).each do |field,elem|
+  #      elem = prepare_entity(elem, field, entity_options)
+  #      yield(elem)
+  #      elem
+  #    end
+  #  else
+  #    named_array_clean_each &block
+  #  end
+  #end
+
+  def each(&block)
     if defined?(Entity) and not @fields.nil? and not @fields.empty?
       @fields.zip(self).each do |field,elem|
         elem = prepare_entity(elem, field, entity_options)
@@ -92,14 +115,30 @@ module NamedArray
         elem
       end
     else
-      named_array_clean_each &block
+      super &block
     end
+
   end
 
-  def named_array_collect
+  #def named_array_collect
+  #  res = []
+
+  #  each do |elem|
+  #    if block_given?
+  #      res << yield(elem)
+  #    else
+  #      res << elem
+  #    end
+  #  end
+
+  #  res
+  #end
+
+
+  def collect
     res = []
 
-    named_array_each do |elem|
+    each do |elem|
       if block_given?
         res << yield(elem)
       else
@@ -110,14 +149,27 @@ module NamedArray
     res
   end
 
-  def named_array_set_brackets(key,value)
-    named_array_clean_set_brackets(Misc.field_position(fields, key), value)
+  #def named_array_set_brackets(key,value)
+  #  named_array_clean_set_brackets(Misc.field_position(fields, key), value)
+  #end
+  
+  def []=(key, value)
+    super(Misc.field_position(fields, key), value)
   end
 
-  def named_array_values_at(*keys)
+  #def named_array_values_at(*keys)
+  #  keys = keys.collect{|k| Misc.field_position(fields, k, true) }
+  #  keys.collect{|k|
+  #    named_array_get_brackets(k) unless k.nil?
+  #  }
+  #end
+
+
+
+  def values_at(*keys)
     keys = keys.collect{|k| Misc.field_position(fields, k, true) }
     keys.collect{|k|
-      named_array_get_brackets(k) unless k.nil?
+      self[k] unless k.nil?
     }
   end
 

data/lib/rbbt/util/open.rb

@@ -11,6 +11,19 @@ module Open
   REMOTE_CACHEDIR = "/tmp/open_cache" 
   FileUtils.mkdir REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
 
+  class << self
+    attr_accessor :repository_dirs
+
+    def repository_dirs
+      @repository_dirs ||= begin
+                             File.exists?(Rbbt.etc.repository_dirs.find) ? 
+                               File.read(Rbbt.etc.repository_dirs.find).split("\n") :
+                               []
+                           end
+    end
+
+  end
+
   def self.cachedir=(cachedir)
     REMOTE_CACHEDIR.replace cachedir
     FileUtils.mkdir REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
@@ -119,15 +132,115 @@ module Open
       CMD.cmd("grep #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
     end
   end
-  
+
+  def self.get_repo_from_dir(dir)
+    @repos ||= {}
+    @repos[dir] ||= begin
+                      repo_path = File.join(dir, '.file_repo')
+                      Persist.open_tokyocabinet(repo_path, false, :clean,TokyoCabinet::BDB )
+                    end
+  end
+
+  def self.get_stream_from_repo(dir, sub_path)
+    repo = get_repo_from_dir(dir)
+    repo.read
+    StringIO.new repo[sub_path]
+  end
+
+  def self.save_content_in_repo(dir, sub_path, content)
+    repo = get_repo_from_dir(dir)
+    repo.write
+    repo[sub_path] = content
+  end
+
+  def self.remove_from_repo(dir, sub_path, recursive = false)
+    repo = get_repo_from_dir(dir)
+    repo.write
+    if recursive
+      repo.outlist repo.range sub_path, true, sub_path.sub(/.$/,('\1'.ord + 1).chr), false
+    else
+      repo.outlist sub_path
+    end
+  end
+
+  def self.exists_in_repo(dir, sub_path, content)
+    repo = get_repo_from_dir(dir)
+    repo.include? sub_path
+  end
+
+  def self.find_repo_dir(file)
+    self.repository_dirs.each do |dir|
+      if file.start_with? dir
+        sub_path = file.to_s[dir.length..-1]
+        return [dir, sub_path]
+      end
+    end
+    nil
+  end
+
+  def self.rm(file)
+    if (dir_sub_path = find_repo_dir(file))
+      remove_from_repo(*dir_sub_path)
+    else
+      FileUtils.rm(file)
+    end
+  end
+
+  def self.rm_rf(file)
+    if (dir_sub_path = find_repo_dir(file))
+      remove_from_repo(*dir_sub_path, true)
+    else
+      FileUtils.rm_rf(file)
+    end
+  end
+
   def self.file_open(file, grep, mode = 'r', invert_grep = false)
+    if (dir_sub_path = find_repo_dir(file))
+      stream =  get_stream_from_repo(*dir_sub_path)
+    else
+      stream =  File.open(file, mode)
+    end
+
     if grep
-      grep(File.open(file, mode), grep, invert_grep)
+      grep(stream, grep, invert_grep)
     else
-      File.open(file, mode)
+      stream
     end
   end
 
+  def self.file_write(file, content, mode = 'w')
+    if (dir_sub_path = find_repo_dir(file))
+      dir_sub_path.push content
+      save_content_in_repo(*dir_sub_path)
+    else
+      File.open(file, mode) do |f|
+        f.flock(File::LOCK_EX)
+        f.write content 
+        f.flock(File::LOCK_UN)
+      end
+    end
+  end
+
+  def self.exists?(file)
+    if (dir_sub_path = find_repo_dir(file))
+      dir_sub_path.push file
+      exists_in_repo(*dir_sub_path)
+    else
+      File.exists? file
+    end
+  end
+
+  def self.lock(file, &block)
+    if (dir_sub_path = find_repo_dir(file))
+      dir, sub_path = dir_sub_path
+      repo = get_repo_from_dir(dir)
+      Misc.lock_in_repo(repo, sub_path, &block)
+    else
+      Misc.lock(file, &block)
+    end
+  end
+
+
   # Decompression
    
   def self.gunzip(stream)
@@ -180,7 +293,15 @@ module Open
   end
 
   def self.open(url, options = {})
-    return url if IO === url
+    if IO === url
+      if block_given?
+        res = yield url 
+        url.close
+        return res
+      else
+        return url 
+      end
+    end
     options = Misc.add_defaults options, :noz => false, :mode => 'r'
 
     mode = Misc.process_options options, :mode
@@ -216,7 +337,9 @@ module Open
     io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
 
     if block_given?
-      yield io 
+      res = yield(io)
+      io.close
+      return res
     else
       io
     end
@@ -269,11 +392,12 @@ module Open
         raise $!
       end
     when String === content
-      File.open(file, mode) do |f|
-        f.flock(File::LOCK_EX)
-        f.write content 
-        f.flock(File::LOCK_UN)
-      end
+      file_write(file, content, mode)
+      #File.open(file, mode) do |f|
+      #  f.flock(File::LOCK_EX)
+      #  f.write content 
+      #  f.flock(File::LOCK_UN)
+      #end
     else
       begin
         File.open(file, mode) do |f|