rbbt-util 5.11.4 → 5.11.5

data/lib/rbbt/util/misc/lock.rb

@@ -0,0 +1,87 @@
+Lockfile.refresh = false if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
+
+module Misc
+
+  LOCK_MUTEX = Mutex.new
+  def self.lock(file, unlock = true)
+    return yield if file.nil?
+    FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
+
+    res = nil
+
+    lock_path = File.expand_path(file + '.lock')
+    lockfile = Lockfile.new(lock_path)
+
+    hostname = Misc.hostname
+    LOCK_MUTEX.synchronize do
+      Misc.insist 2, 0.1 do
+        Misc.insist 3, 0.1 do
+          begin
+            if File.exists? lock_path
+              info = Open.open(lock_path){|f| YAML.load(f) }
+              raise "No info" unless info
+
+              if hostname == info["host"] and not Misc.pid_exists?(info["pid"])
+                Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
+                FileUtils.rm lock_path
+              end
+            end
+          rescue Exception
+            FileUtils.rm lock_path if File.exists? lock_path
+            lockfile = Lockfile.new(lock_path) unless File.exists? lock_path
+            raise $!
+          end
+        end
+      end
+    end
+
+    begin
+      lockfile.lock 
+      res = yield lockfile
+    rescue Lockfile::StolenLockError
+      unlock = false
+    rescue KeepLocked
+      unlock = false
+      res = $!.payload
+    ensure
+      if unlock and lockfile.locked?
+        lockfile.unlock
+      end
+    end
+
+    res
+  end
+  
+
+  LOCK_REPO_SERIALIZER=Marshal
+  def self.lock_in_repo(repo, key, *args)
+    return yield file, *args if repo.nil? or key.nil?
+
+    lock_key = "lock-" << key
+
+    begin
+      if repo[lock_key] and
+        Misc.hostname == (info = LOCK_REPO_SERIALIZER.load(repo[lock_key]))["host"] and 
+        info["pid"] and not Misc.pid_exists?(info["pid"])
+
+        Log.info("Removing lockfile: #{lock_key}. This pid #{Process.pid}. Content: #{info.inspect}")
+        repo.out lock_key 
+      end
+    rescue
+      Log.warn("Error checking lockfile #{lock_key}: #{$!.message}. Removing. Content: #{begin repo[lock_key] rescue "Could not open file" end}")
+      repo.out lock_key if repo.include? lock_key
+    end
+
+    while repo[lock_key]
+      sleep 1
+    end
+    
+    repo[lock_key] = LOCK_REPO_SERIALIZER.dump({:hostname => Misc.hostname, :pid => Process.pid})
+
+    res = yield lock_key, *args
+
+    repo.delete lock_key
+
+    res
+  end
+end

data/lib/rbbt/util/misc/math.rb

@@ -0,0 +1,32 @@
+
+module Misc
+
+  Log2Multiplier = 1.0 / Math.log(2.0)
+  def self.log2(x)
+    Math.log(x) * Log2Multiplier
+  end
+
+  def self.max(list)
+    max = nil
+    list.each do |v|
+      next if v.nil?
+      max = v if max.nil? or v > max
+    end
+    max
+  end
+
+  def self.sum(list)
+    list.compact.inject(0.0){|acc,e| acc += e}
+  end
+
+  def self.mean(list)
+    sum(list) / list.compact.length
+  end
+
+  def self.sd(list)
+    return nil if list.length < 3
+    mean = mean(list)
+    Math.sqrt(list.compact.inject(0.0){|acc,e| d = e - mean; acc += d * d}) / (list.compact.length - 1)
+  end
+
+end

data/lib/rbbt/util/misc/omics.rb

@@ -0,0 +1,183 @@
+module Misc
+
+  IUPAC2BASE = {
+    "A" => ["A"],
+    "C" => ["C"],
+    "G" => ["G"],
+    "T" => ["T"],
+    "U" => ["U"],
+    "R" => "A or G".split(" or "),
+    "Y" => "C or T".split(" or "),
+    "S" => "G or C".split(" or "),
+    "W" => "A or T".split(" or "),
+    "K" => "G or T".split(" or "),
+    "M" => "A or C".split(" or "),
+    "B" => "C or G or T".split(" or "),
+    "D" => "A or G or T".split(" or "),
+    "H" => "A or C or T".split(" or "),
+    "V" => "A or C or G".split(" or "),
+    "N" => %w(A C T G),
+  }
+
+  BASE2COMPLEMENT = {
+    "A" => "T",
+    "C" => "G",
+    "G" => "C",
+    "T" => "A",
+    "U" => "A",
+  }
+
+  THREE_TO_ONE_AA_CODE = {
+    "ala" =>   "A",
+    "arg" =>   "R",
+    "asn" =>   "N",
+    "asp" =>   "D",
+    "cys" =>   "C",
+    "glu" =>   "E",
+    "gln" =>   "Q",
+    "gly" =>   "G",
+    "his" =>   "H",
+    "ile" =>   "I",
+    "leu" =>   "L",
+    "lys" =>   "K",
+    "met" =>   "M",
+    "phe" =>   "F",
+    "pro" =>   "P",
+    "ser" =>   "S",
+    "thr" =>   "T",
+    "trp" =>   "W",
+    "tyr" =>   "Y",
+    "val" =>   "V"
+  }
+  CODON_TABLE = {
+    "ATT" => "I",
+    "ATC" => "I",
+    "ATA" => "I",
+    "CTT" => "L",
+    "CTC" => "L",
+    "CTA" => "L",
+    "CTG" => "L",
+    "TTA" => "L",
+    "TTG" => "L",
+    "GTT" => "V",
+    "GTC" => "V",
+    "GTA" => "V",
+    "GTG" => "V",
+    "TTT" => "F",
+    "TTC" => "F",
+    "ATG" => "M",
+    "TGT" => "C",
+    "TGC" => "C",
+    "GCT" => "A",
+    "GCC" => "A",
+    "GCA" => "A",
+    "GCG" => "A",
+    "GGT" => "G",
+    "GGC" => "G",
+    "GGA" => "G",
+    "GGG" => "G",
+    "CCT" => "P",
+    "CCC" => "P",
+    "CCA" => "P",
+    "CCG" => "P",
+    "ACT" => "T",
+    "ACC" => "T",
+    "ACA" => "T",
+    "ACG" => "T",
+    "TCT" => "S",
+    "TCC" => "S",
+    "TCA" => "S",
+    "TCG" => "S",
+    "AGT" => "S",
+    "AGC" => "S",
+    "TAT" => "Y",
+    "TAC" => "Y",
+    "TGG" => "W",
+    "CAA" => "Q",
+    "CAG" => "Q",
+    "AAT" => "N",
+    "AAC" => "N",
+    "CAT" => "H",
+    "CAC" => "H",
+    "GAA" => "E",
+    "GAG" => "E",
+    "GAT" => "D",
+    "GAC" => "D",
+    "AAA" => "K",
+    "AAG" => "K",
+    "CGT" => "R",
+    "CGC" => "R",
+    "CGA" => "R",
+    "CGG" => "R",
+    "AGA" => "R",
+    "AGG" => "R",
+    "TAA" => "*",
+    "TAG" => "*",
+    "TGA" => "*",
+  }
+
+  def self.correct_icgc_mutation(pos, ref, mut_str)
+    mut = mut_str
+    mut = '-' * (mut_str.length - 1) if mut =~/^-[ACGT]/
+      mut = "+" << mut if ref == '-'
+    [pos, [mut]]
+  end
+
+  def self.correct_vcf_mutation(pos, ref, mut_str)
+    muts = mut_str.nil? ? [] : mut_str.split(',')
+
+    while ref.length >= 1 and muts.reject{|m| m[0] == ref[0]}.empty?
+      ref = ref[1..-1]
+      pos = pos + 1
+      muts = muts.collect{|m| m[1..-1]}
+    end
+
+    muts = muts.collect do |m|
+      case
+      when ref.empty?
+        "+" << m
+      when (m.length < ref.length and (m.empty? or ref.index(m)))
+        "-" * (ref.length - m.length)
+      when (ref.length == 1 and m.length == 1)
+        m
+      else
+        Log.debug{"Cannot understand: #{[ref, m]} (#{ muts })"}
+        '-' * ref.length + m
+      end
+    end
+
+    [pos, muts]
+  end
+
+  def self.IUPAC_to_base(iupac)
+    IUPAC2BASE[iupac]
+  end
+
+
+  def self.sort_mutations(mutations)
+    mutations.collect do |mutation|
+      chr,pos,mut = mutation.split ":"
+      chr.sub!(/^chr/i,'')
+      chr = 22 if chr == "Y"
+      chr = 23 if chr == "X"
+      chr = 24 if chr == "MT" or chr == "M"
+      [chr.to_i, pos.to_i, mut, mutation]
+    end.sort do |a,b|
+      case a[0] <=> b[0]
+      when -1
+        -1
+      when 1
+        1
+      when 0
+        case a[1] <=> b[1]
+        when -1
+          -1
+        when 1
+          1
+        when 0
+          a[2] <=> b[2]
+        end
+      end
+    end.collect{|p| p.last }
+  end
+end

data/lib/rbbt/util/misc/pipes.rb

@@ -0,0 +1,224 @@
+module Misc
+
+  PIPE_MUTEX = Mutex.new
+
+  OPEN_PIPE_IN = []
+  def self.pipe
+    OPEN_PIPE_IN.delete_if{|pipe| pipe.closed? }
+    PIPE_MUTEX.synchronize do
+      sout, sin = IO.pipe
+      OPEN_PIPE_IN << sin
+
+      [sout, sin]
+    end
+  end
+  
+  def self.release_pipes(*pipes)
+    PIPE_MUTEX.synchronize do
+      pipes.flatten.each do |pipe|
+        pipe.close unless pipe.closed?
+      end
+    end
+  end
+
+
+  def self.purge_pipes(*save)
+    PIPE_MUTEX.synchronize do
+      OPEN_PIPE_IN.each do |pipe|
+        next if save.include? pipe
+        pipe.close unless pipe.closed?
+      end
+    end
+  end
+
+  def self.open_pipe(do_fork = false, close = true)
+    raise "No block given" unless block_given?
+
+    sout, sin = Misc.pipe
+
+    if do_fork
+      parent_pid = Process.pid
+      pid = Process.fork {
+        purge_pipes(sin)
+        sout.close
+        begin
+          yield sin
+        rescue
+          Log.exception $!
+          Process.kill :INT, parent_pid
+          Kernel.exit! -1
+        ensure
+          sin.close if close and not sin.closed? 
+        end
+        Kernel.exit! 0
+      }
+      sin.close #if close
+      ConcurrentStream.setup sout, :pids => [pid]
+    else
+      thread = Thread.new(Thread.current) do |parent|
+        begin
+          yield sin
+        rescue
+          parent.raise $!
+        ensure
+          sin.close if close and not sin.closed?
+        end
+      end
+      ConcurrentStream.setup sout, :threads => [thread]
+    end
+    sout
+  end
+
+  def self.tee_stream_fork(stream)
+    stream_out1, stream_in1 = Misc.pipe
+    stream_out2, stream_in2 = Misc.pipe
+
+    splitter_pid = Process.fork do
+      Misc.purge_pipes(stream_in1, stream_in2)
+      stream_out1.close
+      stream_out2.close
+      begin
+        filename = stream.respond_to?(:filename)? stream.filename : nil
+        skip1 = skip2 = false
+        while block = stream.read(2048)
+          begin stream_in1.write block; rescue Exception;  Log.exception $!; skip1 = true end unless skip1 
+          begin stream_in2.write block; rescue Exception;  Log.exception $!; skip2 = true end unless skip2 
+        end
+        raise "Error writing in stream_in2" if skip2
+        raise "Error writing in stream_in2" if skip2
+      rescue Aborted
+        stream.abort if stream.respond_to? :abort
+        raise $!
+      rescue IOError
+        Log.exception $!
+      rescue Exception
+        Log.exception $!
+      ensure
+        stream_in1.close 
+        stream_in2.close 
+        stream.join if stream.respond_to? :join
+      end
+    end
+    stream.close
+    stream_in1.close
+    stream_in2.close
+    #stream.join if stream.respond_to? :join
+
+    ConcurrentStream.setup stream_out1, :pids => [splitter_pid]
+    ConcurrentStream.setup stream_out2, :pids => [splitter_pid]
+
+    [stream_out1, stream_out2]
+  end
+
+  def self.tee_stream_thread(stream)
+    stream_out1, stream_in1 = Misc.pipe
+    stream_out2, stream_in2 = Misc.pipe
+
+    splitter_thread = Thread.new(Thread.current, stream_in1, stream_in2) do |parent,stream_in1,stream_in2|
+      begin
+        filename = stream.respond_to?(:filename)? stream.filename : nil
+        skip1 = skip2 = false
+        while block = stream.read(2048)
+          begin stream_in1.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip1 = true end unless skip1 
+          begin stream_in2.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip2 = true end unless skip2 
+        end
+      rescue Aborted
+        stream.abort if stream.respond_to? :abort
+        raise $!
+      rescue IOError
+        Log.exception $!
+      rescue Exception
+        Log.exception $!
+        parent.raise $!
+      ensure
+        stream_in1.close 
+        stream_in2.close 
+        stream.join if stream.respond_to? :join
+      end
+    end
+
+    ConcurrentStream.setup stream_out1, :threads => splitter_thread
+    ConcurrentStream.setup stream_out2, :threads => splitter_thread
+
+    [stream_out1, stream_out2]
+  end
+
+  class << self
+    alias tee_stream tee_stream_thread 
+  end
+
+  def self.read_full_stream(io)
+    str = ""
+    begin
+      while block = io.read(2048)
+        str << block
+      end
+    rescue
+      io.abort if io.respond_to? :abort
+    ensure
+      io.join if io.respond_to? :join
+      io.close if io.respond_to? :close
+    end
+    str
+  end
+
+  def self.consume_stream(io)
+    begin
+      while block = io.read(2048)
+        return if io.eof?
+        Thread.pass 
+     end
+    rescue
+      io.abort if io.respond_to? :abort
+    ensure
+      io.join if io.respond_to? :join
+      io.close if io.respond_to? :close
+    end
+  end
+
+  def self.read_stream(stream, size)
+    str = nil
+    Thread.pass while IO.select([stream],nil,nil,1).nil?
+    while not str = stream.read(size)
+      IO.select([stream],nil,nil,1) 
+      Thread.pass
+      raise ClosedStream if stream.eof?
+    end
+
+    while str.length < size
+      raise ClosedStream if stream.eof?
+      IO.select([stream],nil,nil,1)
+      if new = stream.read(size-str.length)
+        str << new
+      end
+    end
+    str
+  end
+
+  def self.read_stream(stream, size)
+    str = nil
+    Thread.pass while IO.select([stream],nil,nil,1).nil?
+    while not str = stream.read(size)
+      IO.select([stream],nil,nil,1) 
+      Thread.pass
+      raise ClosedStream if stream.eof?
+    end
+
+    while str.length < size
+      raise ClosedStream if stream.eof?
+      IO.select([stream],nil,nil,1)
+      if new = stream.read(size-str.length)
+        str << new
+      end
+    end
+    str
+  end
+  def self._read_stream(stream, size)
+    str = ""
+    while (len=str.length) < size
+      str << (stream.read(size-len) or break)
+    end
+    str
+  end
+
+end