rbbt-util 5.7.0 → 5.8.0

data/lib/rbbt/util/concurrency/processes/socket_old.rb

@@ -0,0 +1,144 @@
+class RbbtProcessQueue
+  class RbbtProcessSocket
+
+    class ClosedSocket < Exception; end
+
+    attr_accessor :sin, :sout, :in_lockfile, :out_lockfile
+    def initialize(lockfile = nil)
+      @sout, @sin = File.pipe
+
+      lockfile ||= TmpFile.tmp_file
+
+      @lockfile = lockfile
+      @in_lockfile = lockfile + '.in'
+      @out_lockfile = lockfile + '.out'
+      raise "in_lockfile exists?" if File.exists? @in_lockfile
+      raise "out_lockfile exists?" if File.exists? @in_lockfile
+      FileUtils.touch @in_lockfile
+      FileUtils.touch @out_lockfile
+    end
+
+    def self.serialize(obj)
+      dump = nil
+      begin
+        case obj
+        when TSV
+          type = "T"
+          info = obj.info
+          info.delete_if{|k,v| v.nil?}
+          dump = Marshal.dump([info, {}.merge(obj)])
+        else
+          type = "M"
+          dump = Marshal.dump(obj)
+        end
+        payload = [type, dump].pack('A1a*')
+        length = payload.bytesize
+        #Log.info "Writing #{ length }"
+        [length].pack('L') << payload
+      rescue Exception
+        Log.error "Serialize error for: #{Misc.fingerprint obj} - #{Misc.fingerprint dump}"
+        raise $!
+      end
+    end
+
+    def self.unserialize(str)
+      begin
+        c, dump = str.unpack("A1a*")
+        case c
+        when "M"
+          return Marshal.load(dump)
+        when "T"
+          info, hash = Marshal.load(dump)
+          return TSV.setup(hash, info)
+        end
+      rescue Exception
+        Log.error "Unserialize error for: #{Misc.fingerprint str}"
+        raise $!
+      end
+    end
+
+    def read_sout(length)
+      str = ""
+      str << sout.readpartial(length-str.length) while str.length < length
+      str
+    end
+
+    def write_sin(str)
+      str_length = str.length
+      wrote = 0
+      wrote += sin.write_nonblock(str[wrote..-1]) while wrote < str_length
+    end
+
+    def push(obj)
+      Filelock in_lockfile do
+        payload = RbbtProcessSocket.serialize(obj)
+        sin << payload
+      end
+    end
+
+
+    def pop
+      r = []
+
+      payload = begin
+                  Filelock out_lockfile do
+                    raise ClosedQueue if sout.eof?
+                    r,w,e = IO.select([sout], [], [], 1)
+                    raise TryAgain if r.empty?
+
+                    first_char = read_sout(4)
+                    length = first_char.unpack('L').first
+                    #Log.info "Reading #{ length }"
+                    read_sout(length)
+                  end
+                rescue TryAgain
+                  sleep 1
+                end
+
+      RbbtProcessSocket.unserialize(payload)
+    end
+
+    def pop
+      loop do
+        r,w,e = IO.select([sout], [], [], 1)
+        next if r.empty?
+        break
+      end
+
+      first_char = read_sout(4)
+      length = first_char.unpack('L').first
+      #Log.info "Reading #{ length }"
+      read_sout(length)
+    end
+  rescue TryAgain
+    sleep 1
+  end
+
+      RbbtProcessSocket.unserialize(payload)
+    end
+
+    def rest
+      sin.close
+      str = sout.read
+      res = []
+
+      while not str.empty?
+        first_char = str[0]
+        next if first_char.nil?
+        length = first_char.unpack("C").first
+        dump = str[1..length]
+        res << Marshal.load(dump)
+        str = str[length+1..-1]
+      end
+
+      res
+    end
+
+    def clean
+      FileUtils.rm @in_lockfile if File.exists? @in_lockfile
+      FileUtils.rm @out_lockfile if File.exists? @out_lockfile
+      sin.close unless sin.closed?
+      sout.close unless sout.closed?
+    end
+  end
+end

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -0,0 +1,53 @@
+require 'rbbt/util/concurrency/processes/socket'
+class RbbtProcessQueue
+  class RbbtProcessQueueWorker
+    attr_accessor :pid, :queue, :callback_queue, :block
+    def initialize(queue, callback_queue = nil, &block)
+      @queue, @callback_queue, @block = queue, callback_queue, block
+
+      @pid = Process.fork do
+        begin
+          @queue.swrite.close
+          @callback_queue.sread.close if @callback_queue
+
+          Signal.trap(:INT){ raise Aborted; }
+          loop do
+            p = @queue.pop
+            raise p if Exception === p
+            res = @block.call p
+            @callback_queue.push res if @callback_queue
+          end
+
+          exit 0
+        rescue ClosedStream
+          exit 0
+        rescue Aborted
+          exit -1
+        rescue Exception
+          Log.exception $!
+          @callback_queue.push($!) if @callback_queue
+          exit -1
+        end
+
+      end
+    end
+
+    def join
+      Process.waitpid @pid
+    end
+
+    def abort
+      Process.kill :INT, @pid
+    end
+
+    def done?
+      begin
+        Process.waitpid @pid, Process::WNOHANG
+      rescue Errno::ECHILD
+        true
+      rescue
+        false
+      end
+    end
+  end
+end

data/lib/rbbt/util/concurrency/threads.rb

@@ -0,0 +1,76 @@
+class RbbtThreadQueue
+  attr_accessor :num_threads, :threads, :queue, :mutex, :block, :done
+
+  class RbbtThreadQueueWorker < Thread
+    def initialize(queue, mutex = nil, &block)
+      if mutex.nil?
+        super(Thread.current) do |current|
+          begin
+            loop do
+              p = queue.pop
+              block.call *p
+            end
+          rescue Exception
+            current.raise $! unless Aborted === $!
+          end
+        end
+      else
+        super(Thread.current) do |current|
+          begin
+            loop do
+              p = queue.pop
+              p << mutex 
+              block.call *p
+            end
+          rescue Exception
+            current.raise $! unless Aborted === $!
+          end
+        end
+      end
+    end
+
+    def clean
+      raise Aborted if alive?
+    end
+  end
+
+  def initialize(num_threads)
+    @num_threads = num_threads
+    @threads = []
+    @queue = Queue.new
+    @mutex = Mutex.new
+  end
+
+  def init(use_mutex = false, &block)
+    clean
+    num_threads.times do |i|
+      @threads << RbbtThreadQueueWorker.new(queue, use_mutex ? mutex : nil, &block)
+    end
+  end
+
+  def join
+    while queue.length > 0 or queue.num_waiting < @threads.length
+      Thread.pass 
+      raise "No worker thread survived" if @threads.empty? and queue.length > 0
+    end
+    @threads.delete_if{|t| t.alive?}
+    @threads.each{|t| t.raise Aborted } 
+    @threads.each{|t| t.join(0.1) } 
+  end
+
+  def clean
+    threads.each{ |t| t.clean }.clear
+  end
+
+  def process(e)
+    queue << e
+  end
+
+
+  def self.each(list, num = 3, &block)
+    q = RbbtThreadQueue.new num
+    q.init(&block)
+    list.each do |elem| q.process elem end
+    q.join
+  end
+end

data/lib/rbbt/util/log.rb

@@ -126,6 +126,14 @@ def ppp(message)
   puts ""
 end
 
+def fff(object)
+  stack = caller
+  Log.debug{"#{Log.color :cyan, "FINGERPRINT:"} " << stack.first}
+  Log.debug{""}
+  Log.debug{require 'rbbt/util/misc'; "=> " << Misc.fingerprint(object) }
+  Log.debug{""}
+end
+
 def ddd(message, file = $stdout)
   stack = caller
   Log.debug{"#{Log.color :cyan, "DEBUG:"} " << stack.first}
@@ -134,12 +142,36 @@ def ddd(message, file = $stdout)
   Log.debug{""}
 end
 
-def fff(object)
+def lll(message, file = $stdout)
   stack = caller
-  Log.debug{"#{Log.color :cyan, "FINGERPRINT:"} " << stack.first}
-  Log.debug{""}
-  Log.debug{require 'rbbt/util/misc'; "=> " << Misc.fingerprint(object) }
-  Log.debug{""}
+  Log.low{"#{Log.color :cyan, "LOW:"} " << stack.first}
+  Log.low{""}
+  Log.low{"=> " << message.inspect}
+  Log.low{""}
+end
+
+def mmm(message, file = $stdout)
+  stack = caller
+  Log.low{"#{Log.color :cyan, "MEDIUM:"} " << stack.first}
+  Log.low{""}
+  Log.low{"=> " << message.inspect}
+  Log.low{""}
+end
+
+def hhh(message, file = $stdout)
+  stack = caller
+  Log.high{"#{Log.color :cyan, "HIGH:"} " << stack.first}
+  Log.high{""}
+  Log.high{"=> " << message.inspect}
+  Log.high{""}
+end
+
+def iii(message, file = $stdout)
+  stack = caller
+  Log.info{"#{Log.color :cyan, "INFO:"} " << stack.first}
+  Log.info{""}
+  Log.info{"=> " << message.inspect}
+  Log.info{""}
 end
 
 

data/lib/rbbt/util/misc.rb

@@ -16,8 +16,11 @@ class Hash
   end
 end
 
-class ParameterException < Exception; end
+class ParameterException < StandardError; end
 class FieldNotFoundError < StandardError;end
+class Aborted < Exception; end
+class TryAgain < Exception; end
+class ClosedStream < Exception; end
 
 module LaterString
   def to_s
@@ -28,18 +31,34 @@ end
 Lockfile.refresh = false if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
 module Misc
 
+  def self.read_stream(stream, size)
+    str = nil
+    while not str = stream.read(size)
+      IO.select([stream],nil,nil,1) 
+      raise ClosedStream if stream.eof?
+    end
+
+    while str.length < size
+      raise ClosedStream if stream.eof?
+      IO.select([stream],nil,nil,1)
+      if new = stream.read(size-str.length)
+        str << new
+      end
+    end
+    str
+  end
   def self.parse_cmd_params(str)
     return str if Array === str
     str.scan(/
-      (?:["']([^"']*?)["']) |
-      ([^"'\s]+)
+             (?:["']([^"']*?)["']) |
+             ([^"'\s]+)
     /x).flatten.compact
   end
 
   def self.correct_icgc_mutation(pos, ref, mut_str)
     mut = mut_str
     mut = '-' * (mut_str.length - 1) if mut =~/^-[ACGT]/
-    mut = "+" << mut if ref == '-'
+      mut = "+" << mut if ref == '-'
     [pos, [mut]]
   end
 
@@ -504,6 +523,72 @@ end
     "tyr" =>   "Y",
     "val" =>   "V"
   }
+  CODON_TABLE = {
+    "ATT" => "I",
+    "ATC" => "I",
+    "ATA" => "I",
+    "CTT" => "L",
+    "CTC" => "L",
+    "CTA" => "L",
+    "CTG" => "L",
+    "TTA" => "L",
+    "TTG" => "L",
+    "GTT" => "V",
+    "GTC" => "V",
+    "GTA" => "V",
+    "GTG" => "V",
+    "TTT" => "F",
+    "TTC" => "F",
+    "ATG" => "M",
+    "TGT" => "C",
+    "TGC" => "C",
+    "GCT" => "A",
+    "GCC" => "A",
+    "GCA" => "A",
+    "GCG" => "A",
+    "GGT" => "G",
+    "GGC" => "G",
+    "GGA" => "G",
+    "GGG" => "G",
+    "CCT" => "P",
+    "CCC" => "P",
+    "CCA" => "P",
+    "CCG" => "P",
+    "ACT" => "T",
+    "ACC" => "T",
+    "ACA" => "T",
+    "ACG" => "T",
+    "TCT" => "S",
+    "TCC" => "S",
+    "TCA" => "S",
+    "TCG" => "S",
+    "AGT" => "S",
+    "AGC" => "S",
+    "TAT" => "Y",
+    "TAC" => "Y",
+    "TGG" => "W",
+    "CAA" => "Q",
+    "CAG" => "Q",
+    "AAT" => "N",
+    "AAC" => "N",
+    "CAT" => "H",
+    "CAC" => "H",
+    "GAA" => "E",
+    "GAG" => "E",
+    "GAT" => "D",
+    "GAC" => "D",
+    "AAA" => "K",
+    "AAG" => "K",
+    "CGT" => "R",
+    "CGC" => "R",
+    "CGA" => "R",
+    "CGG" => "R",
+    "AGA" => "R",
+    "AGG" => "R",
+    "TAA" => "*",
+    "TAG" => "*",
+    "TGA" => "*",
+  }
 
   #def self.fast_align(reference, sequence)
   #