rbbt-util 5.13.37 → 5.14.0

data/lib/rbbt/util/R.rb

@@ -38,7 +38,7 @@ source('#{UTIL}');
 
   def self.interactive(script, options = {})
     TmpFile.with_file do |init_file|
-        Open.write(init_file) do |f|
+       Open.write(init_file) do |f|
           f.puts "# Loading basic rbbt environment"
           f.puts "library(utils);\n"
           f.puts "source('#{R::UTIL}');\n"

data/lib/rbbt/util/cmd.rb

@@ -112,7 +112,6 @@ module CMD
       stderr = Log::HIGH
     end
 
-    # Process cmd_options
     cmd_options = process_cmd_options options
     if cmd =~ /'\{opt\}'/
       cmd.sub!('\'{opt}\'', cmd_options) 
@@ -203,7 +202,6 @@ module CMD
           Log.log line, stderr if Integer === stderr and log
         end
         serr.close
-        Thread.exit!
       end
 
       #SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
@@ -217,7 +215,6 @@ module CMD
           err << serr.gets if Integer === stderr
         end
         serr.close
-        Thread.exit
       end
 
       ConcurrentStream.setup sout, :pids => [pid], :autojoin => true

data/lib/rbbt/util/concurrency/processes.rb

@@ -96,6 +96,9 @@ class RbbtProcessQueue
     begin
       @process_monitor.join
       close_callback if @callback
+    rescue Aborted
+      Log.error "Aborted joining queue"
+      raise $!
     rescue Exception
       Log.error "Exception joining queue: #{$!.message}"
       raise $!

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -36,7 +36,6 @@ class RbbtProcessQueue
         rescue Aborted, Interrupt
           Log.warn "Worker #{Process.pid} aborted"
           Kernel.exit! 0
-          #Kernel.exit! -1
         rescue Exception
           Log.exception $!
           @callback_queue.push($!) if @callback_queue

data/lib/rbbt/util/log.rb

@@ -2,10 +2,18 @@ require 'term/ansicolor'
 require 'rbbt/util/color'
 require 'rbbt/util/log/progress'
 
+class MockMutex
+  def synchronize
+    yield
+  end
+end
+
 module Log
   extend Term::ANSIColor
 
 
+  #ToDo: I'm not sure if using a Mutex here really gives troubles in CPU concurrency
+  #LOG_MUTEX = MockMutex.new
   LOG_MUTEX = Mutex.new
 
   SEVERITY_NAMES ||= begin
@@ -16,8 +24,16 @@ module Log
                      names
                    end
 
+  def self.last_caller(stack)
+    line = nil
+    while line.nil? or line =~ /util\/log\.rb/ and stack.any?
+      line = stack.shift 
+    end
+    line ||= caller.first
+  end
+
   def self.ignore_stderr
-    #LOG_MUTEX.synchronize do
+    LOG_MUTEX.synchronize do
       backup_stderr = STDERR.dup
       File.open('/dev/null', 'w') do |f|
         STDERR.reopen(f)
@@ -28,7 +44,7 @@ module Log
           backup_stderr.close
         end
       end
-    #end
+    end
   end
 
   def self.get_level(level)
@@ -133,22 +149,18 @@ module Log
     message = "" << highlight << message << color(0) if severity >= INFO
     str = prefix << " " << message
 
-    #LOG_MUTEX.synchronize do
-    STDERR.puts str
-    Log::LAST.replace "log"
-    logfile.puts str unless logfile.nil?
-    nil
-    #end
+    LOG_MUTEX.synchronize do
+      STDERR.puts str
+      Log::LAST.replace "log"
+      logfile.puts str unless logfile.nil?
+      nil
+    end
   end
 
   def self.log_obj_inspect(obj, level, file = $stdout)
     stack = caller
 
-    line = nil
-    while line.nil? or line =~ /util\/log\.rb/ and stack.any?
-      line = stack.shift 
-    end
-    line ||= caller.first
+    line = Log.last_caller stack
 
     level = Log.get_level level
     name = Log::SEVERITY_NAMES[level] + ": "
@@ -161,11 +173,7 @@ module Log
   def self.log_obj_fingerprint(obj, level, file = $stdout)
     stack = caller
 
-    line = nil
-    while line.nil? or line =~ /util\/log\.rb/ and stack.any?
-      line = stack.shift 
-    end
-    line ||= caller.first
+    line = Log.last_caller stack
 
     level = Log.get_level level
     name = Log::SEVERITY_NAMES[level] + ": "
@@ -210,6 +218,25 @@ module Log
     error("BACKTRACE:\n" + e.backtrace * "\n") 
   end
 
+  def self.color_stack(stack)
+    stack.collect do |line|
+      line = line.sub('`',"'")
+      color = :green if line =~ /workflow/
+      color = :blue if line =~ /rbbt-/
+      Log.color color, line
+    end
+  end
+
+  def self.stack(stack)
+    LOG_MUTEX.synchronize do
+
+      STDERR.puts Log.color :magenta, "Stack trace: " << Log.last_caller(caller)
+      color_stack(stack).each do |line|
+        STDERR.puts line
+      end
+    end
+  end
+
   case ENV['RBBT_LOG'] 
   when 'DEBUG' 
     self.severity = DEBUG

data/lib/rbbt/util/log/progress/report.rb

@@ -10,11 +10,11 @@ module Log
 
     def print(io, str)
       return if ENV["RBBT_NO_PROGRESS"] == "true"
-      #LOG_MUTEX.synchronize do
+      LOG_MUTEX.synchronize do
         STDERR.print str
         Log.logfile.puts str unless Log.logfile.nil?
         Log::LAST.replace "progress"
-      #end
+      end
     end
 
     attr_accessor :history, :mean_max
@@ -90,6 +90,7 @@ module Log
       if Log::LAST != "progress"
         length = Log::ProgressBar.cleanup_bars
         bars = BARS
+        print(io, Log.color(:yellow, "...Progress\n"))
         bars.sort_by{|b| b.depth }.reverse.each do |bar|
           print(io, Log.color(:yellow ,bar.report_msg) << "\n")
         end

data/lib/rbbt/util/log/progress/util.rb

@@ -31,7 +31,7 @@ module Log
     end
 
     def self.remove_bar(bar)
-      bar.done 
+      bar.done if bar.respond_to? :done
       BAR_MUTEX.synchronize do
         REMOVE << bar
       end

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -82,18 +82,18 @@ module ConcurrentStream
     join_callback
 
     @joined = true
-    close unless closed?
     lockfile.unlock if lockfile and lockfile.locked?
+    close unless closed?
   end
 
-  def abort_threads
+  def abort_threads(exception)
     Log.medium "Aborting threads (#{Thread.current.inspect}) #{@threads.collect{|t| t.inspect } * ", "}"
 
     @threads.each do |t| 
       @aborted = false if t == Thread.current
       next if t == Thread.current
       Log.medium "Aborting thread #{t.inspect}"
-      t.raise Aborted.new 
+      t.raise exception ? exception : Aborted.new 
     end if @threads
 
     sleeped = false
@@ -117,11 +117,16 @@ module ConcurrentStream
   end
 
   def abort_pids
-    @pids.each{|pid| Process.kill :INT, pid } if @pids
+    @pids.each do |pid|
+      begin 
+        Process.kill :INT, pid 
+      rescue Errno::ESRCH
+      end
+    end if @pids
     @pids = []
   end
 
-  def abort
+  def abort(exception = nil)
     return if @aborted
     Log.medium "Aborting stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"
     @aborted = true 
@@ -131,8 +136,9 @@ module ConcurrentStream
       @abort_callback = nil
       close unless closed?
 
-      abort_threads
+      abort_threads(exception)
       abort_pids
+    ensure
       lockfile.unlock if lockfile and lockfile.locked?
     end
     Log.medium "Aborted stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"

data/lib/rbbt/util/misc/development.rb

@@ -135,24 +135,30 @@ module Misc
     try = 0
     begin
       yield
-    rescue Aborted
+    rescue TryAgain
+      sleep sleep
+      retry
+    rescue Aborted, Interrupt
       if msg
         Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
       else
         Log.warn("Not Insisting after Aborted: #{$!.message}")
       end
+      raise $!
     rescue Exception
       if msg
-        Log.warn("Insisting after exception: #{$!.message} -- #{msg}")
+        Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
       else
-        Log.warn("Insisting after exception: #{$!.message}")
-      end
+        Log.warn("Insisting after exception:  #{$!.class} #{$!.message}")
+      end 
+
       if sleep and try > 0
         sleep sleep
         sleep = sleep_array.shift if sleep_array
       else
         Thread.pass
       end
+
       try += 1
       retry if try < times
       raise $!

data/lib/rbbt/util/misc/lock.rb

@@ -28,11 +28,11 @@ module Misc
 
     res = nil
 
-    lock_path = File.expand_path(file + '.lock')
     if options[:lock]
       lockfile = options[:lock]
       lockfile.lock unless lockfile.locked?
     else
+      lock_path = File.expand_path(file + '.lock')
       lockfile = Lockfile.new(lock_path, options)
       lockfile.lock 
     end

data/lib/rbbt/util/misc/omics.rb

@@ -125,9 +125,11 @@ module Misc
 
   def self.correct_vcf_mutation(pos, ref, mut_str)
     muts = mut_str.nil? ? [] : mut_str.split(',')
+    muts.collect!{|m| m == '<DEL>' ? '-' : m }
 
     while ref.length >= 1 and muts.reject{|m| m[0] == ref[0]}.empty?
       ref = ref[1..-1]
+      raise "REF nil" if ref.nil?
       pos = pos + 1
       muts = muts.collect{|m| m[1..-1]}
     end

data/lib/rbbt/util/misc/pipes.rb

@@ -175,7 +175,7 @@ module Misc
       Log.medium "Consuming stream #{Misc.fingerprint io}"
       begin
         into.sync == true if IO === into
-        while not io.closed? and block = io.read(2048 * 10)
+        while not io.closed? and block = io.read(2048)
           into << block if into
         end
         io.join if io.respond_to? :join
@@ -213,48 +213,47 @@ module Misc
   end
 
   def self.sensiblewrite(path, content = nil, options = {}, &block)
+    force = Misc.process_options options, :force
     lock_options = Misc.pull_keys options, :lock
     lock_options = lock_options[:lock] if Hash === lock_options[:lock]
-    return if Open.exists? path
+    return if Open.exists? path and not force
     tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_dir})
     tmp_path_lock = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_lock_dir})
     Misc.lock tmp_path_lock, lock_options do
-      return if Open.exists? path
-      if not Open.exists? path
-        FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory? File.dirname(tmp_path)
-        FileUtils.rm_f tmp_path if File.exists? tmp_path
-        begin
-          case
-          when block_given?
-            File.open(tmp_path, 'wb', &block)
-          when String === content
-            File.open(tmp_path, 'wb') do |f| f.write content end
-          when (IO === content or StringIO === content or File === content)
-
-            Open.write(tmp_path) do |f|
-              f.sync = true
-              while block = content.read(2048)
-                f.write block
-              end
+      return if Open.exists? path and not force
+      FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory? File.dirname(tmp_path)
+      FileUtils.rm_f tmp_path if File.exists? tmp_path
+      begin
+        case
+        when block_given?
+          File.open(tmp_path, 'wb', &block)
+        when String === content
+          File.open(tmp_path, 'wb') do |f| f.write content end
+        when (IO === content or StringIO === content or File === content)
+
+          Open.write(tmp_path) do |f|
+            f.sync = true
+            while block = content.read(2048)
+              f.write block
             end
-          else
-            File.open(tmp_path, 'wb') do |f|  end
           end
-
-          Open.mv tmp_path, path, lock_options
-          content.join if content.respond_to? :join
-        rescue Aborted
-          Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
-          content.abort if content.respond_to? :abort
-          Open.rm path if File.exists? path
-        rescue Exception
-          Log.medium "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
-          content.abort if content.respond_to? :abort
-          Open.rm path if File.exists? path
-          raise $!
-        ensure
-          FileUtils.rm_f tmp_path if File.exists? tmp_path
+        else
+          File.open(tmp_path, 'wb') do |f|  end
         end
+
+        Open.mv tmp_path, path, lock_options
+        content.join if content.respond_to? :join
+      rescue Aborted
+        Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
+        content.abort if content.respond_to? :abort
+        Open.rm path if File.exists? path
+      rescue Exception
+        Log.medium "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
+        content.abort if content.respond_to? :abort
+        Open.rm path if File.exists? path
+        raise $!
+      ensure
+        FileUtils.rm_f tmp_path if File.exists? tmp_path
       end
     end
   end
@@ -269,7 +268,7 @@ module Misc
     end
   end
 
-  def self.sort_stream(stream, header_hash = "#", cmd_args = nil)
+  def self.sort_stream(stream, header_hash = "#", cmd_args = " -u ")
     Misc.open_pipe do |sin|
       begin
         if defined? Step and Step === stream
@@ -343,64 +342,68 @@ module Misc
     end
   end
 
-  def self.paste_streams(streams, lines = nil, sep = "\t", header = nil)
-    sep ||= "\t"
-    num_streams = streams.length
-    Misc.open_pipe do |sin|
-      sin.puts header if header
-      streams = streams.collect do |stream|
-        if defined? Step and Step === stream
-          stream.get_stream || stream.join.path.open
-        else
-          stream
-        end
+  def self._paste_streams(streams, output, lines = nil, sep = "\t", header = nil)
+    output.puts header if header
+    streams = streams.collect do |stream|
+      if defined? Step and Step === stream
+        stream.get_stream || stream.join.path.open
+      else
+        stream
       end
+    end
 
-      begin
-        done_streams = []
-        lines ||= streams.collect{|s| s.gets }
-        keys = []
-        parts = []
-        lines.each_with_index do |line,i|
-          key, *p = line.strip.split(sep, -1) 
-          keys[i] = key
-          parts[i] = p
-        end
-        sizes = parts.collect{|p| p.length }
-        last_min = nil
-        while lines.compact.any?
-          min = keys.compact.sort.first
-          str = []
-          keys.each_with_index do |key,i|
-            case key
-            when min
-              str << [parts[i] * sep]
-              line = lines[i] = streams[i].gets
-              if line.nil?
-                keys[i] = nil
-                parts[i] = nil
-              else
-                k, *p = line.strip.split(sep, -1)
-                keys[i] = k
-                parts[i] = p
-              end
+    begin
+      done_streams = []
+      lines ||= streams.collect{|s| s.gets }
+      keys = []
+      parts = []
+      lines.each_with_index do |line,i|
+        key, *p = line.strip.split(sep, -1) 
+        keys[i] = key
+        parts[i] = p
+      end
+      sizes = parts.collect{|p| p.length }
+      last_min = nil
+      while lines.compact.any?
+        min = keys.compact.sort.first
+        str = []
+        keys.each_with_index do |key,i|
+          case key
+          when min
+            str << [parts[i] * sep]
+            line = lines[i] = streams[i].gets
+            if line.nil?
+              keys[i] = nil
+              parts[i] = nil
             else
-              str << [sep * (sizes[i]-1)] if sizes[i] > 0
+              k, *p = line.strip.split(sep, -1)
+              keys[i] = k
+              parts[i] = p
             end
+          else
+            str << [sep * (sizes[i]-1)] if sizes[i] > 0
           end
-
-          sin.puts [min, str*sep] * sep
-        end
-        streams.each do |stream|
-          stream.join if stream.respond_to? :join
         end
-      rescue 
-        Log.exception $!
-        streams.each do |stream|
-          stream.abort if stream.respond_to? :abort
-        end
-        raise $!
+
+        output.puts [min, str*sep] * sep
       end
+      streams.each do |stream|
+        stream.join if stream.respond_to? :join
+      end
+    rescue 
+      Log.exception $!
+      streams.each do |stream|
+        stream.abort if stream.respond_to? :abort
+      end
+      raise $!
+    end
+  end
+
+  def self.paste_streams(streams, lines = nil, sep = "\t", header = nil)
+    sep ||= "\t"
+    num_streams = streams.length
+    Misc.open_pipe do |sin|
+      self._paste_streams(streams, sin, lines, sep, header)
     end
   end