rbbt-util 5.11.4 → 5.11.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1b14c182984b18f946f50ba4d5e07fafec05f8b5
-  data.tar.gz: f6cdb39396d9bd4871e74ab0eb63015373e415d9
+  metadata.gz: 6ec4bc56a7d0d260d93be40ee61d23c08f58d5a8
+  data.tar.gz: 6f51a747302e936ed1e5953e929e5120fa5caa89
 SHA512:
-  metadata.gz: a51d7e12fb6fa32385092372830f5f3349fd697c0263f561c287cf17860b604c9544254444d0148c214381639f76e4de396b6d32c1d4a3cbf0e8bd631b45f3c2
-  data.tar.gz: d56ed3b1b3a048d5a68f1ad5b17f1548ed993387ff5fc7313b79716faeb31a96e5e7aa289b3a0d7821aeaa60ffbe2767ac771f9aba57c31b990b94186b1a77dc
+  metadata.gz: db5a97b2787f182444de536059beb027d038573c0c3ac3196ec44ce18d2561fa3204a38229d3054e5c92594309fc4a488cc6e9116e370c4f2d383b28a8ed9e71
+  data.tar.gz: 8a4fc523ae9266ceea9964f26f85834a8f5c0e0899a523bb938e2e813da191651215ec356432ee7b467f39b470328b32dda911d1b2cd9bd757fa467ad14d54ce

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -235,7 +235,8 @@ module TSV
         store.add *value
       when IO
         return if value.nil?
-        store.puts value.strip
+        value.strip!
+        store.puts value
       else
         store << value
       end 

data/lib/rbbt/util/cmd.rb

@@ -72,7 +72,7 @@ module CMD
     def read(*args)
       data = original_read(*args) unless self.closed? or self.eof?
 
-      self.close if self.eof? and not self.closed?
+      self.close if not self.closed? and self.eof?
 
       data || ""
     end

data/lib/rbbt/util/misc.rb

@@ -5,6 +5,17 @@ require 'cgi'
 require 'zlib'
 require 'rubygems/package'
 require 'rbbt/util/tar'
+require 'rbbt/util/misc/exceptions'
+require 'rbbt/util/misc/concurrent_stream'
+require 'rbbt/util/misc/indiferent_hash'
+require 'rbbt/util/misc/pipes'
+require 'rbbt/util/misc/format'
+require 'rbbt/util/misc/omics'
+require 'rbbt/util/misc/inspect'
+require 'rbbt/util/misc/math'
+require 'rbbt/util/misc/development'
+require 'rbbt/util/misc/lock'
+
 
 class Hash
   def chunked_values_at(keys, max = 5000)
@@ -16,372 +27,14 @@ class Hash
   end
 end
 
-class ParameterException < Exception; end
-class FieldNotFoundError < Exception;end
-class Aborted < Exception; end
-class TryAgain < Exception; end
-class ClosedStream < Exception; end
-class KeepLocked < Exception
-  attr_accessor :payload
-  def initialize(payload)
-    @payload = payload
-  end
-end
-
 module LaterString
   def to_s
     yield
   end
 end
 
-module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
-
-  def joined?
-    @joined
-  end
-
-  def join
-
-    if @threads and @threads.any?
-      @threads.each do |t| 
-        t.join 
-      end
-      @threads = []
-    end
-
-    if @pids and @pids.any?
-      @pids.each do |pid| 
-        begin
-          Process.waitpid(pid, Process::WUNTRACED)
-          raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
-        rescue Errno::ECHILD
-        end
-      end 
-      @pids = []
-    end
-
-    if @callback and not joined?
-      @callback.call
-      @callback = nil
-    end
-
-    @joined = true
-  end
-
-  def abort
-    @threads.each{|t| t.raise Aborted.new } if @threads
-    @threads.each{|t| t.join } if @threads
-    @pids.each{|pid| Process.kill :INT, pid } if @pids
-    @pids.each{|pid| Process.waitpid pid } if @pids
-    @abort_callback.call if @abort_callback
-    @abort_callback = nil
-  end
-
-  def self.setup(stream, options = {}, &block)
-    threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
-    stream.extend ConcurrentStream unless ConcurrentStream === stream
-
-    stream.threads ||= []
-    stream.pids ||= []
-    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
-    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
-
-    callback = block if block_given?
-    if stream.callback and callback
-      old_callback = stream.callback
-      stream.callback = Proc.new do
-        old_callback.call
-        callback.call
-      end
-    else
-      stream.callback = callback
-    end
-
-    stream.filename = filename unless filename.nil?
-
-    stream
-  end
-end
-
-
-Lockfile.refresh = false if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
 module Misc
 
-
-  PIPE_MUTEX = Mutex.new
-
-  OPEN_PIPE_IN = []
-  def self.pipe
-    OPEN_PIPE_IN.delete_if{|pipe| pipe.closed? }
-    PIPE_MUTEX.synchronize do
-      sout, sin = IO.pipe
-      OPEN_PIPE_IN << sin
-
-      [sout, sin]
-    end
-  end
-  
-  def self.release_pipes(*pipes)
-    PIPE_MUTEX.synchronize do
-      pipes.flatten.each do |pipe|
-        pipe.close unless pipe.closed?
-      end
-    end
-  end
-
-
-  def self.purge_pipes(*save)
-    PIPE_MUTEX.synchronize do
-      OPEN_PIPE_IN.each do |pipe|
-        next if save.include? pipe
-        pipe.close unless pipe.closed?
-      end
-    end
-  end
-
-  def self.open_pipe(do_fork = false, close = true)
-    raise "No block given" unless block_given?
-
-    sout, sin = Misc.pipe
-
-    if do_fork
-      parent_pid = Process.pid
-      pid = Process.fork {
-        purge_pipes(sin)
-        sout.close
-        begin
-          yield sin
-        rescue
-          Log.exception $!
-          Process.kill :INT, parent_pid
-          Kernel.exit! -1
-        ensure
-          sin.close if close and not sin.closed? 
-        end
-        Kernel.exit! 0
-      }
-      sin.close #if close
-      ConcurrentStream.setup sout, :pids => [pid]
-    else
-      thread = Thread.new(Thread.current) do |parent|
-        begin
-          yield sin
-        rescue
-          parent.raise $!
-        ensure
-          sin.close if close and not sin.closed?
-        end
-      end
-      ConcurrentStream.setup sout, :threads => [thread]
-    end
-    sout
-  end
-
-  def self.tee_stream_fork(stream)
-    stream_out1, stream_in1 = Misc.pipe
-    stream_out2, stream_in2 = Misc.pipe
-
-    splitter_pid = Process.fork do
-      Misc.purge_pipes(stream_in1, stream_in2)
-      stream_out1.close
-      stream_out2.close
-      begin
-        filename = stream.respond_to?(:filename)? stream.filename : nil
-        skip1 = skip2 = false
-        while block = stream.read(2048)
-          begin stream_in1.write block; rescue Exception;  Log.exception $!; skip1 = true end unless skip1 
-          begin stream_in2.write block; rescue Exception;  Log.exception $!; skip2 = true end unless skip2 
-        end
-        raise "Error writing in stream_in2" if skip2
-        raise "Error writing in stream_in2" if skip2
-      rescue Aborted
-        stream.abort if stream.respond_to? :abort
-        raise $!
-      rescue IOError
-        Log.exception $!
-      rescue Exception
-        Log.exception $!
-      ensure
-        stream_in1.close 
-        stream_in2.close 
-        stream.join if stream.respond_to? :join
-      end
-    end
-    stream.close
-    stream_in1.close
-    stream_in2.close
-    #stream.join if stream.respond_to? :join
-
-    ConcurrentStream.setup stream_out1, :pids => [splitter_pid]
-    ConcurrentStream.setup stream_out2, :pids => [splitter_pid]
-
-    [stream_out1, stream_out2]
-  end
-
-  def self.tee_stream_thread(stream)
-    stream_out1, stream_in1 = Misc.pipe
-    stream_out2, stream_in2 = Misc.pipe
-
-    splitter_thread = Thread.new(Thread.current, stream_in1, stream_in2) do |parent,stream_in1,stream_in2|
-      begin
-        filename = stream.respond_to?(:filename)? stream.filename : nil
-        skip1 = skip2 = false
-        while block = stream.read(2048)
-          begin stream_in1.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip1 = true end unless skip1 
-          begin stream_in2.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip2 = true end unless skip2 
-        end
-      rescue Aborted
-        stream.abort if stream.respond_to? :abort
-        raise $!
-      rescue IOError
-        Log.exception $!
-      rescue Exception
-        Log.exception $!
-        parent.raise $!
-      ensure
-        stream_in1.close 
-        stream_in2.close 
-        stream.join if stream.respond_to? :join
-      end
-    end
-
-    ConcurrentStream.setup stream_out1, :threads => splitter_thread
-    ConcurrentStream.setup stream_out2, :threads => splitter_thread
-
-    [stream_out1, stream_out2]
-  end
-
-  class << self
-    alias tee_stream tee_stream_thread 
-  end
-
-  def self.read_full_stream(io)
-    str = ""
-    begin
-      while block = io.read(2048)
-        str << block
-      end
-    rescue
-      io.abort if io.respond_to? :abort
-    ensure
-      io.join if io.respond_to? :join
-      io.close if io.respond_to? :close
-    end
-    str
-  end
-
-  def self.consume_stream(io)
-    begin
-      Thread.pass while block = io.read(2048)
-    rescue
-      io.abort if io.respond_to? :abort
-    ensure
-      io.join if io.respond_to? :join
-      io.close if io.respond_to? :close
-    end
-  end
-
-  def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
-    i = 0
-    re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
-      text.split(re).collect do |paragraph|
-      i += 1
-      str = if i % 2 == 1
-              words = paragraph.gsub(/\s+/, "\s").split(" ")
-              lines = []
-              line = " "*offset
-              word = words.shift
-              while word
-                word = word[0..size-indent-offset-4] + '...' if word.length >= size - indent - offset
-                while word and Log.uncolor(line).length + Log.uncolor(word).length <= size - indent
-                  line << word << " "
-                  word = words.shift
-                end
-                lines << ((" " * indent) << line[0..-2])
-                line = ""
-              end
-              (lines * "\n")
-            else
-              paragraph
-            end
-      offset = 0
-      str
-      end*""
-  end
-
-  def self.format_definition_list_item(dt, dd, size = 80, indent = 20, color = :yellow)
-    dd = "" if dd.nil?
-    dt = dt.to_s + ":" unless dd.empty?
-    dt = Log.color color, dt if color
-    len = Log.uncolor(dt).length
-
-    if indent < 0
-      text = format_paragraph(dd, size, indent.abs+1, 0)
-      text = dt << "\n" << text
-    else
-      offset = len - indent
-      offset = 0 if offset < 0
-      text = format_paragraph(dd, size, indent.abs+1, offset)
-      text[0..len-1] = dt
-    end
-    text
-  end
-
-  def self.format_definition_list(defs, size = 80, indent = 20, color = :yellow)
-    entries = []
-    defs.each do |dt,dd|
-      text = format_definition_list_item(dt,dd,size,indent,color)
-      entries << text
-    end
-    entries * "\n\n"
-  end
-
-  def self.read_stream(stream, size)
-    str = nil
-    Thread.pass while IO.select([stream],nil,nil,1).nil?
-    while not str = stream.read(size)
-      IO.select([stream],nil,nil,1) 
-      Thread.pass
-      raise ClosedStream if stream.eof?
-    end
-
-    while str.length < size
-      raise ClosedStream if stream.eof?
-      IO.select([stream],nil,nil,1)
-      if new = stream.read(size-str.length)
-        str << new
-      end
-    end
-    str
-  end
-
-  def self.read_stream(stream, size)
-    str = nil
-    Thread.pass while IO.select([stream],nil,nil,1).nil?
-    while not str = stream.read(size)
-      IO.select([stream],nil,nil,1) 
-      Thread.pass
-      raise ClosedStream if stream.eof?
-    end
-
-    while str.length < size
-      raise ClosedStream if stream.eof?
-      IO.select([stream],nil,nil,1)
-      if new = stream.read(size-str.length)
-        str << new
-      end
-    end
-    str
-  end
-  def self._read_stream(stream, size)
-    str = ""
-    while (len=str.length) < size
-      str << (stream.read(size-len) or break)
-    end
-    str
-  end
-
   def self.parse_cmd_params(str)
     return str if Array === str
     str.scan(/
@@ -390,39 +43,6 @@ module Misc
     /x).flatten.compact
   end
 
-  def self.correct_icgc_mutation(pos, ref, mut_str)
-    mut = mut_str
-    mut = '-' * (mut_str.length - 1) if mut =~/^-[ACGT]/
-      mut = "+" << mut if ref == '-'
-    [pos, [mut]]
-  end
-
-  def self.correct_vcf_mutation(pos, ref, mut_str)
-    muts = mut_str.nil? ? [] : mut_str.split(',')
-
-    while ref.length >= 1 and muts.reject{|m| m[0] == ref[0]}.empty?
-      ref = ref[1..-1]
-      pos = pos + 1
-      muts = muts.collect{|m| m[1..-1]}
-    end
-
-    muts = muts.collect do |m|
-      case
-      when ref.empty?
-        "+" << m
-      when (m.length < ref.length and (m.empty? or ref.index(m)))
-        "-" * (ref.length - m.length)
-      when (ref.length == 1 and m.length == 1)
-        m
-      else
-        Log.debug{"Cannot understand: #{[ref, m]} (#{ muts })"}
-        '-' * ref.length + m
-      end
-    end
-
-    [pos, muts]
-  end
-
   def self.pid_exists?(pid)
     return false if pid.nil?
     begin
@@ -433,25 +53,6 @@ module Misc
     end
   end
 
-  COLOR_LIST = %w(#BC80BD #CCEBC5 #FFED6F #8DD3C7 #FFFFB3 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9)
-
-  def self.colors_for(list)
-    unused = COLOR_LIST.dup
-
-    used = {}
-    colors = list.collect do |elem|
-      if used.include? elem
-        used[elem]
-      else
-        color = unused.shift
-        used[elem]=color
-        color
-      end
-    end
-
-    [colors, used]
-  end
-
   def self.collapse_ranges(ranges)
     processed = []
     last = nil
@@ -527,10 +128,6 @@ module Misc
     end
   end
 
-  Log2Multiplier = 1.0 / Math.log(2.0)
-  def self.log2(x)
-    Math.log(x) * Log2Multiplier
-  end
 
   def self.prepare_entity(entity, field, options = {})
     return entity unless defined? Entity
@@ -555,116 +152,6 @@ module Misc
     entity
   end
  
-  ARRAY_MAX_LENGTH = 1000
-  STRING_MAX_LENGTH = ARRAY_MAX_LENGTH * 10
-
-  def self.sanitize_filename(filename, length = 254)
-    if filename.length > length
-      if filename =~ /(\..{2,9})$/
-        extension = $1
-      else
-        extension = ''
-      end
-
-      post_fix = "--#{filename.length}@#{length}_#{Misc.digest(filename)[0..4]}" + extension
-
-      filename = filename[0..(length - post_fix.length - 1)] << post_fix
-    else
-      filename
-    end
-    filename
-  end
-
-  def self.fingerprint(obj)
-    case obj
-    when nil
-      "nil"
-    when (defined? Step and Step)
-      obj.path || Misc.fingerprint([obj.task.name, obj.inputs])
-    when TrueClass
-      "true"
-    when FalseClass
-      "false"
-    when Symbol
-      ":" << obj.to_s
-    when String
-      if obj.length > 100
-        "'" << obj.slice(0,20) << "<...#{obj.length}...>" << obj.slice(-10,10) << " " << "'"
-      else 
-        "'" << obj << "'"
-      end
-    when (defined? AnnotatedArray and AnnotatedArray)
-      "<A: #{fingerprint Annotated.purge(obj)} #{fingerprint obj.info}>"
-    when (defined? TSV and TSV::Parser)
-      "<TSVStream:" + obj.filename + "--" << Misc.fingerprint(obj.options) << ">"
-    when IO
-      "<IO:" + (obj.respond_to?(:filename) ? obj.filename : obj.inspect) + ">"
-    when File
-      "<File:" + obj.path + ">"
-    when Array
-      if (length = obj.length) > 10
-        "[#{length}--" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
-      else
-        "[" << (obj.collect{|e| fingerprint(e) } * ",") << "]"
-      end
-    when (defined? TSV and TSV)
-      obj.with_unnamed do
-        "TSV:{"<< fingerprint(obj.all_fields|| []).inspect << ";" << fingerprint(obj.keys).inspect << "}"
-      end
-    when Hash
-      if obj.length > 10
-        "H:{"<< fingerprint(obj.keys) << ";" << fingerprint(obj.values) << "}"
-      else
-        new = "{"
-        obj.each do |k,v|
-          new << k.to_s << '=>' << fingerprint(v) << ' '
-        end
-        if new.length > 1
-           new[-1] =  "}"
-        else
-          new << '}'
-        end
-        new
-      end
-    else
-      obj.to_s
-    end
-  end
-
-
-  def self.remove_long_items(obj)
-    case
-    when IO === obj
-      remove_long_items("IO: " + obj.filename)
-    when obj.respond_to?(:path)
-      remove_long_items("File: " + obj.path)
-    when TSV::Parser === obj
-      remove_long_items("TSV Stream: " + obj.filename + " -- " << Misc.fingerprint(obj.options))
-    when TSV === obj
-      remove_long_items((obj.all_fields || []) + obj.keys.sort)
-    when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
-      remove_long_items(obj[0..ARRAY_MAX_LENGTH-2] << "TRUNCATED at #{ ARRAY_MAX_LENGTH } (#{obj.length})")
-    when (Hash === obj and obj.length > ARRAY_MAX_LENGTH)
-      remove_long_items(obj.collect.compact[0..ARRAY_MAX_LENGTH-2] << ["TRUNCATED", "at #{ ARRAY_MAX_LENGTH } (#{obj.length})"])
-    when (String === obj and obj.length > STRING_MAX_LENGTH)
-      obj[0..STRING_MAX_LENGTH-1] << " TRUNCATED at #{STRING_MAX_LENGTH} (#{obj.length})"
-    when Hash === obj
-      new = {}
-      obj.each do |k,v|
-        new[k] = remove_long_items(v)
-      end
-      new
-    when Array === obj
-      obj.collect do |e| remove_long_items(e) end
-    else
-      obj
-    end
-  end
-
-  #def self.remove_long_items(obj)
-  #  return fingerprint(obj)
-  #end
-
   def self.ensembl_server(organism)
     date = organism.split("/")[1]
     if date.nil?
@@ -674,18 +161,6 @@ module Misc
     end
   end
 
-  def self.filename?(string)
-    String === string and string.length > 0 and string.length < 250 and File.exists?(string)
-  end
-
-  def self.max(list)
-    max = nil
-    list.each do |v|
-      next if v.nil?
-      max = v if max.nil? or v > max
-    end
-    max
-  end
 
   def self.google_venn(list1, list2, list3, name1 = nil, name2 = nil, name3 = nil, total = nil)
     name1 ||= "list 1"
@@ -710,20 +185,6 @@ module Misc
     url = "https://chart.googleapis.com/chart?cht=v&chs=500x300&chd=t:#{sizes * ","}&chco=FF6342,ADDE63,63C6DE,FFFFFF&chdl=#{label}"
   end
 
-  def self.sum(list)
-    list.compact.inject(0.0){|acc,e| acc += e}
-  end
-
-  def self.mean(list)
-    sum(list) / list.compact.length
-  end
-
-  def self.sd(list)
-    return nil if list.length < 3
-    mean = mean(list)
-    Math.sqrt(list.compact.inject(0.0){|acc,e| d = e - mean; acc += d * d}) / (list.compact.length - 1)
-  end
-
   def self.consolidate(list)
     list.inject(nil){|acc,e|
       if acc.nil?
@@ -795,208 +256,6 @@ end
     proportions
   end
 
-  IUPAC2BASE = {
-    "A" => ["A"],
-    "C" => ["C"],
-    "G" => ["G"],
-    "T" => ["T"],
-    "U" => ["U"],
-    "R" => "A or G".split(" or "),
-    "Y" => "C or T".split(" or "),
-    "S" => "G or C".split(" or "),
-    "W" => "A or T".split(" or "),
-    "K" => "G or T".split(" or "),
-    "M" => "A or C".split(" or "),
-    "B" => "C or G or T".split(" or "),
-    "D" => "A or G or T".split(" or "),
-    "H" => "A or C or T".split(" or "),
-    "V" => "A or C or G".split(" or "),
-    "N" => %w(A C T G),
-  }
-
-  BASE2COMPLEMENT = {
-    "A" => "T",
-    "C" => "G",
-    "G" => "C",
-    "T" => "A",
-    "U" => "A",
-  }
-
-  THREE_TO_ONE_AA_CODE = {
-    "ala" =>   "A",
-    "arg" =>   "R",
-    "asn" =>   "N",
-    "asp" =>   "D",
-    "cys" =>   "C",
-    "glu" =>   "E",
-    "gln" =>   "Q",
-    "gly" =>   "G",
-    "his" =>   "H",
-    "ile" =>   "I",
-    "leu" =>   "L",
-    "lys" =>   "K",
-    "met" =>   "M",
-    "phe" =>   "F",
-    "pro" =>   "P",
-    "ser" =>   "S",
-    "thr" =>   "T",
-    "trp" =>   "W",
-    "tyr" =>   "Y",
-    "val" =>   "V"
-  }
-  CODON_TABLE = {
-    "ATT" => "I",
-    "ATC" => "I",
-    "ATA" => "I",
-    "CTT" => "L",
-    "CTC" => "L",
-    "CTA" => "L",
-    "CTG" => "L",
-    "TTA" => "L",
-    "TTG" => "L",
-    "GTT" => "V",
-    "GTC" => "V",
-    "GTA" => "V",
-    "GTG" => "V",
-    "TTT" => "F",
-    "TTC" => "F",
-    "ATG" => "M",
-    "TGT" => "C",
-    "TGC" => "C",
-    "GCT" => "A",
-    "GCC" => "A",
-    "GCA" => "A",
-    "GCG" => "A",
-    "GGT" => "G",
-    "GGC" => "G",
-    "GGA" => "G",
-    "GGG" => "G",
-    "CCT" => "P",
-    "CCC" => "P",
-    "CCA" => "P",
-    "CCG" => "P",
-    "ACT" => "T",
-    "ACC" => "T",
-    "ACA" => "T",
-    "ACG" => "T",
-    "TCT" => "S",
-    "TCC" => "S",
-    "TCA" => "S",
-    "TCG" => "S",
-    "AGT" => "S",
-    "AGC" => "S",
-    "TAT" => "Y",
-    "TAC" => "Y",
-    "TGG" => "W",
-    "CAA" => "Q",
-    "CAG" => "Q",
-    "AAT" => "N",
-    "AAC" => "N",
-    "CAT" => "H",
-    "CAC" => "H",
-    "GAA" => "E",
-    "GAG" => "E",
-    "GAT" => "D",
-    "GAC" => "D",
-    "AAA" => "K",
-    "AAG" => "K",
-    "CGT" => "R",
-    "CGC" => "R",
-    "CGA" => "R",
-    "CGG" => "R",
-    "AGA" => "R",
-    "AGG" => "R",
-    "TAA" => "*",
-    "TAG" => "*",
-    "TGA" => "*",
-  }
-
-  #def self.fast_align(reference, sequence)
-  #
-  #require 'narray'
-  #  init_gap = -1
-  #  gap = -2
-  #  diff = -2
-  #  same = 2
-
-  #  cols = sequence.length + 1
-  #  rows = reference.length + 1
-
-  #  a = NArray.int(cols, rows)
-
-  #  for spos in 0..cols-1 do a[spos, 0] = spos * init_gap end
-  #  for rpos in 0..rows-1 do a[0, rpos] = rpos * init_gap end
-
-  #  spos = 1
-  #  while spos < cols do
-  #    rpos = 1
-  #    while rpos < rows do
-  #      match = a[spos-1,rpos-1] + (sequence[spos-1] != reference[rpos-1] ? diff : same)
-  #      skip_sequence = a[spos-1,rpos] + gap
-  #      skip_reference = a[spos,rpos-1] + gap
-  #      a[spos,rpos] = [match, skip_sequence, skip_reference].max
-  #      rpos += 1
-  #    end
-  #    spos += 1
-  #  end
-
-  #  start = Misc.max(a[-1,0..rows-1])
-  #  start_pos = a[-1,0..rows-1].to_a.index start
-
-  #  ref = ''
-  #  seq = ''
-  #  rpos = start_pos
-  #  spos = cols - 1
-
-  #  while spos > 0 and rpos > 0
-  #    score = a[spos,rpos]
-  #    score_match = a[spos-1,rpos-1]
-  #    score_skip_reference = a[spos,rpos-1]
-  #    score_skip_sequence = a[spos-1,rpos]
-
-  #    case
-  #    when score == score_match + (sequence[spos-1] != reference[rpos-1] ? diff : same)
-  #      ref << reference[rpos-1]
-  #      seq << sequence[spos-1]
-  #      spos -= 1
-  #      rpos -= 1
-  #    when score == score_skip_reference + gap
-  #      ref << reference[rpos-1]
-  #      seq << '-'
-  #      rpos -= 1
-  #    when score == score_skip_sequence + gap
-  #      seq << sequence[spos-1]
-  #      ref << '-'
-  #      spos -= 1
-  #    else
-  #      raise "stop"
-  #    end
-  #  end
-
-  #  while (rpos > 0)
-  #    ref << reference[rpos-1]
-  #    seq = seq << '-'
-  #    rpos -= 1    
-  #  end
-
-  #  while (spos > 0)
-  #    seq << sequence[spos-1]
-  #    ref = ref + '-'
-  #    spos -= 1
-  #  end
-  #  
-  #  [ref.reverse + reference[start_pos..-1], seq.reverse + '-' * (rows - start_pos - 1)]
-  #end
-
-  def self.IUPAC_to_base(iupac)
-    IUPAC2BASE[iupac]
-  end
-
-  def self.is_filename?(string)
-    return true if string.respond_to? :exists
-    return true if String === string and string.length < 265 and File.exists? string
-    return false
-  end
 
   def self.sorted_array_hits(a1, a2)
     e1, e2 = a1.shift, a2.shift
@@ -1132,100 +391,6 @@ end
       end
   end
 
-  def self.benchmark(repeats = 1, message = nil)
-    require 'benchmark'
-    res = nil
-    begin
-      measure = Benchmark.measure do
-        repeats.times do
-          res = yield
-        end
-      end
-      if message
-        puts "#{message }: #{ repeats } repeats"
-      else
-        puts "Benchmark for #{ repeats } repeats"
-      end
-      puts measure
-    rescue Exception
-      puts "Benchmark aborted"
-      raise $!
-    end
-    res
-  end
-
-  def self.profile_html(options = {})
-    require 'ruby-prof'
-    RubyProf.start
-    begin
-      res = yield
-    rescue Exception
-      puts "Profiling aborted"
-      raise $!
-    ensure
-      result = RubyProf.stop
-      printer = RubyProf::MultiPrinter.new(result)
-      TmpFile.with_file do |dir|
-        FileUtils.mkdir_p dir unless File.exists? dir
-        printer.print(:path => dir, :profile => 'profile')
-        CMD.cmd("firefox  -no-remote  '#{ dir }'")
-      end
-    end
-
-    res
-  end
-
-  def self.profile_graph(options = {})
-    require 'ruby-prof'
-    RubyProf.start
-    begin
-      res = yield
-    rescue Exception
-      puts "Profiling aborted"
-      raise $!
-    ensure
-      result = RubyProf.stop
-      #result.eliminate_methods!([/annotated_array_clean_/])
-      printer = RubyProf::GraphPrinter.new(result)
-      printer.print(STDOUT, options)
-    end
-
-    res
-  end
-
-  def self.profile(options = {})
-    require 'ruby-prof'
-    RubyProf.start
-    begin
-      res = yield
-    rescue Exception
-      puts "Profiling aborted"
-      raise $!
-    ensure
-      result = RubyProf.stop
-      printer = RubyProf::FlatPrinter.new(result)
-      printer.print(STDOUT, options)
-    end
-
-    res
-  end
-
-  def self.memprof
-    require 'memprof'
-    Memprof.start
-    begin
-      res = yield
-    rescue Exception
-      puts "Profiling aborted"
-      raise $!
-    ensure
-      Memprof.stop
-      print Memprof.stats
-    end
-
-    res
-  end
-
   def self.do_once(&block)
     return nil if $__did_once
     $__did_once = true
@@ -1334,22 +499,6 @@ end
     html
   end
 
-  #def self.path_relative_to(basedir, path)
-  #  path = File.expand_path(path) unless path[0] == "/"
-  #  basedir = File.expand_path(basedir) unless basedir[0] == "/"
-
-  #  basedir << "/" unless basedir[-1] == "/"
-  #  case
-  #  when path == basedir
-  #    "."
-  #  #when path =~ /#{Regexp.quote basedir}\/(.*)/
-  #  when path.index(basedir) == 0
-  #    return path[basedir.length..-1]
-  #  else
-  #    return nil
-  #  end
-  #end
-
   def self.path_relative_to(basedir, path)
     path = File.expand_path(path) unless path[0] == "/"
     basedir = File.expand_path(basedir) unless basedir[0] == "/"
@@ -1369,88 +518,6 @@ end
     @hostanem ||= `hostname`.strip
   end
 
-  LOCK_MUTEX = Mutex.new
-  def self.lock(file, unlock = true)
-    return yield if file.nil?
-    FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
-
-    res = nil
-
-    lock_path = File.expand_path(file + '.lock')
-    lockfile = Lockfile.new(lock_path)
-
-    hostname = Misc.hostname
-    LOCK_MUTEX.synchronize do
-      Misc.insist 2, 0.1 do
-        Misc.insist 3, 0.1 do
-          begin
-            if File.exists? lock_path
-              info = Open.open(lock_path){|f| YAML.load(f) }
-              raise "No info" unless info
-
-              if hostname == info["host"] and not Misc.pid_exists?(info["pid"])
-                Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
-                FileUtils.rm lock_path
-              end
-            end
-          rescue Exception
-            FileUtils.rm lock_path if File.exists? lock_path
-            lockfile = Lockfile.new(lock_path) unless File.exists? lock_path
-            raise $!
-          end
-        end
-      end
-    end
-
-    begin
-      lockfile.lock 
-      res = yield lockfile
-    rescue Lockfile::StolenLockError
-      unlock = false
-    rescue KeepLocked
-      unlock = false
-      res = $!.payload
-    ensure
-      if unlock and lockfile.locked?
-        lockfile.unlock
-      end
-    end
-
-    res
-  end
-  
-
-  LOCK_REPO_SERIALIZER=Marshal
-  def self.lock_in_repo(repo, key, *args)
-    return yield file, *args if repo.nil? or key.nil?
-
-    lock_key = "lock-" << key
-
-    begin
-      if repo[lock_key] and
-        Misc.hostname == (info = LOCK_REPO_SERIALIZER.load(repo[lock_key]))["host"] and 
-        info["pid"] and not Misc.pid_exists?(info["pid"])
-
-        Log.info("Removing lockfile: #{lock_key}. This pid #{Process.pid}. Content: #{info.inspect}")
-        repo.out lock_key 
-      end
-    rescue
-      Log.warn("Error checking lockfile #{lock_key}: #{$!.message}. Removing. Content: #{begin repo[lock_key] rescue "Could not open file" end}")
-      repo.out lock_key if repo.include? lock_key
-    end
-
-    while repo[lock_key]
-      sleep 1
-    end
-    
-    repo[lock_key] = LOCK_REPO_SERIALIZER.dump({:hostname => Misc.hostname, :pid => Process.pid})
-
-    res = yield lock_key, *args
-
-    repo.delete lock_key
-
-    res
-  end
 
   def self.common_path(dir, file)
     file = File.expand_path file
@@ -1479,32 +546,6 @@ end
     res
   end
 
-  def self.to_utf8(string)
-    string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
-  end
-
-  def self.fixutf8(string)
-    return nil if string.nil?
-    return string if (string.respond_to? :valid_encoding? and string.valid_encoding?) or
-    (string.respond_to? :valid_encoding and string.valid_encoding)
-
-    if string.respond_to?(:encode)
-      string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
-    else
-      require 'iconv'
-      @@ic ||= Iconv.new('UTF-8//IGNORE', 'UTF-8')
-      @@ic.iconv(string)
-    end
-  end
-
-  def self.fixascii(string)
-    if string.respond_to?(:encode)
-      self.fixutf8(string).encode("ASCII-8BIT") 
-    else
-      string
-    end
-  end
-
   def self.sensiblewrite(path, content = nil, &block)
     return if File.exists? path
     tmp_path = path + '.sensible_write'
@@ -1556,69 +597,6 @@ end
     new_options
   end
 
-  def self.digest(text)
-    Digest::MD5.hexdigest(text)
-  end
-
-  HASH2MD5_MAX_STRING_LENGTH = 1000
-  HASH2MD5_MAX_ARRAY_LENGTH = 100
-  def self.hash2md5(hash)
-    str = ""
-    keys = hash.keys
-    keys = keys.clean_annotations if keys.respond_to? :clean_annotations
-    keys = keys.sort_by{|k| k.to_s}
-
-    if hash.respond_to? :unnamed
-      unnamed = hash.unnamed
-      hash.unnamed = true 
-    end
-    keys.each do |k|
-      next if k == :monitor or k == "monitor" or k == :in_situ_persistence or k == "in_situ_persistence"
-      v = hash[k]
-      case
-      when TrueClass === v
-        str << k.to_s << "=>true" 
-      when FalseClass === v
-        str << k.to_s << "=>false" 
-      when Hash === v
-        str << k.to_s << "=>" << hash2md5(v)
-      when Symbol === v
-        str << k.to_s << "=>" << v.to_s
-      when (String === v and v.length > HASH2MD5_MAX_STRING_LENGTH)
-        str << k.to_s << "=>" << v[0..HASH2MD5_MAX_STRING_LENGTH] << "; #{ v.length }"
-      when String === v
-        str << k.to_s << "=>" << v
-      when (Array === v and v.length > HASH2MD5_MAX_ARRAY_LENGTH)
-        str << k.to_s << "=>[" << v[0..HASH2MD5_MAX_ARRAY_LENGTH] * "," << "; #{ v.length }]"
-      when TSV::Parser === v
-        str << remove_long_items(v)
-      when Array === v
-        str << k.to_s << "=>[" << v * "," << "]"
-      when File === v
-        str << k.to_s << "=>[File:" << v.path << "]"
-      else
-        v_ins = v.inspect
-
-        case
-        when v_ins =~ /:0x0/
-          str << k.to_s << "=>" << v_ins.sub(/:0x[a-f0-9]+@/,'')
-        else
-          str << k.to_s << "=>" << v_ins
-        end
-
-      end
-
-      str << "_" << hash2md5(v.info) if defined? Annotated and Annotated === v
-    end
-    hash.unnamed = unnamed if hash.respond_to? :unnamed
-
-    if str.empty?
-      ""
-    else
-      digest(str)
-    end
-  end
-
   def self.process_options(hash, *keys)
     if keys.length == 1
       hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s) 
@@ -1815,7 +793,6 @@ end
     chunks
   end
 
-
   def self.append_zipped(current, new)
     current.each do |v|
       n = new.shift
@@ -1832,145 +809,6 @@ end
     return [] if array.empty? or (first = array.first).nil?
     first.zip(*array[1..-1])
   end
-
-  def self.camel_case(string)
-    return string if string !~ /_/ && string =~ /[A-Z]+.*/
-      string.split(/_|(\d+)/).map{|e| 
-        (e =~ /^[A-Z]{2,}$/ ? e : e.capitalize) 
-      }.join
-  end
-
-  def self.camel_case_lower(string)
-      string.split('_').inject([]){ |buffer,e| 
-        buffer.push(buffer.empty? ? e.downcase : (e =~ /^[A-Z]{2,}$/ ? e : e.capitalize)) 
-      }.join
-  end
-
-  def self.snake_case(string)
-    return nil if string.nil?
-    string = string.to_s if Symbol === string
-    string.
-      gsub(/([A-Z]{2,})([A-Z][a-z])/,'\1_\2').
-      gsub(/([a-z])([A-Z])/,'\1_\2').
-      gsub(/\s/,'_').gsub(/[^\w_]/, '').
-      split("_").collect{|p| p.match(/[A-Z]{2,}/) ? p : p.downcase } * "_"
-  end
-
-  # source: https://gist.github.com/ekdevdes/2450285
-  # author: Ethan Kramer (https://github.com/ekdevdes)
-  def self.humanize(value, options = {})
-    if options.empty?
-      options[:format] = :sentence
-    end
-
-    values = []
-    values = value.split('_')
-    values.each_index do |index|
-      # lower case each item in array
-      # Miguel Vazquez edit: Except for acronyms
-      values[index].downcase! unless values[index].match(/[a-zA-Z][A-Z]/)
-    end
-    if options[:format] == :allcaps
-      values.each do |value|
-        value.capitalize!
-      end
-
-      if options.empty?
-        options[:seperator] = " "
-      end
-
-      return values.join " "
-    end
-
-    if options[:format] == :class
-      values.each do |value|
-        value.capitalize!
-      end
-
-      return values.join ""
-    end
-
-    if options[:format] == :sentence
-      values[0].capitalize! unless values[0].match(/[a-zA-Z][A-Z]/)
-
-      return values.join " "
-    end
-
-    if options[:format] == :nocaps
-      return values.join " "
-    end
-  end
-end
-
-#TODO: REMOVE
-#class RBBTError < StandardError
-#  attr_accessor :info
-#
-#  alias old_to_s to_s
-#  def to_s
-#    str = old_to_s.dup
-#    if info
-#      str << "\n" << "Additional Info:\n---\n" << info << "---"
-#    end
-#    str
-#  end
-#end
-
-module IndiferentHash
-
-  def self.setup(hash)
-    hash.extend IndiferentHash 
-  end
-
-  def merge(other)
-    new = self.dup
-    IndiferentHash.setup(new)
-    other.each do |k,value|
-      new.delete k
-      new[k] = value
-    end
-    new
-  end
-
-  def [](key)
-    res = super(key) 
-    return res unless res.nil?
-
-    case key
-    when Symbol, Module
-      super(key.to_s)
-    when String
-      super(key.to_sym)
-    else
-      super(key)
-    end
-  end
-
-  def values_at(*key_list)
-    key_list.inject([]){|acc,key| acc << self[key]}
-  end
-
-  def include?(key)
-    case key
-    when Symbol, Module
-      super(key) || super(key.to_s)
-    when String
-      super(key) || super(key.to_sym)
-    else
-      super(key)
-    end
-  end
-
-  def delete(key)
-    case key
-    when Symbol, Module
-      super(key) || super(key.to_s)
-    when String
-      super(key) || super(key.to_sym)
-    else
-      super(key)
-    end
-  end
 end
 
 module PDF2Text