rbbt-util 5.32.10 → 5.32.16

data/lib/rbbt/util/R.rb

@@ -41,7 +41,7 @@ source('#{UTIL}');
 
     if monitor
       #io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true))
-      io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true, :xvfb => true))
+      io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true, :xvfb => options[:xvfb]))
       while line = io.gets
         case monitor
         when Proc
@@ -52,7 +52,7 @@ source('#{UTIL}');
       end
       nil
     else
-      CMD.cmd('R --no-save --slave --quiet', options.merge(:in => cmd, :xvfb => true))
+      CMD.cmd('R --no-save --slave --quiet', options.merge(:in => cmd, :xvfb => options[:xvfb]))
     end
   end
 

data/lib/rbbt/util/cmd.rb

@@ -192,25 +192,47 @@ module CMD
 
       ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail 
 
-      err_thread = Thread.new do
-        while line = serr.gets
-          bar.process(line) if bar
-          sout.log = line
-          Log.log "STDERR [#{pid}]: " +  line, stderr 
-        end if Integer === stderr and log
-        serr.close
+      if (Integer === stderr and log) || bar
+        err_thread = Thread.new do
+          while line = serr.gets
+            bar.process(line) if bar
+            sout.log = line
+            Log.log "STDERR [#{pid}]: " +  line, stderr if log
+          end 
+          serr.close
+        end
+      else
+        err_thread = Misc.consume_stream(serr, true)
       end
 
       sout.threads = [in_thread, err_thread, wait_thr].compact
 
       sout
     else
-      err = ""
-      err_thread = Thread.new do
-        while not serr.eof?
-          err << serr.gets if Integer === stderr
+
+      if bar
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            line = serr.gets 
+            bar.process(line) 
+            err << line if Integer === stderr and log
+          end
+          serr.close
         end
-        serr.close
+      elsif log and Integer === stderr
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            err << serr.gets 
+          end
+          serr.close
+        end
+      else
+        Misc.consume_stream(serr, true)
+        #serr.close 
+        err_thread = nil
+        err = ""
       end
 
       ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail 
@@ -220,7 +242,11 @@ module CMD
 
       status = wait_thr.value
       if not status.success? and not no_fail
-        raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+        if !err.empty?
+          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+        else
+          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}"
+        end
       else
         Log.log err, stderr if Integer === stderr and log
       end
@@ -263,11 +289,6 @@ module CMD
       if c == "\n"
         bar.process(line) if bar
         starting = true
-        #if pid
-        #  Log.logn "STDOUT [#{pid}]: ", level
-        #else
-        #  Log.logn "STDOUT: ", level
-        #end
         line = "" if bar
       end
     end 

data/lib/rbbt/util/log/progress/report.rb

@@ -12,32 +12,35 @@ module Log
     attr_accessor :history, :mean_max, :max_history
     def thr_msg
       if @history.nil?
-        @history ||= [[@ticks, Time.now] ]
+        @history ||= [[0, @start], [@ticks, Time.now] ]
       elsif @last_ticks != @ticks
         @history << [@ticks, Time.now]
-        max_history ||= case 
-                      when @ticks > 20
-                        count = @ticks - @last_count
-                        count = 1 if count == 0
-                        if @max
-                          times = @max / count
-                          num = times / 20
-                          num = 2 if num < 2
-                        else
-                          num = 10
+        max_history ||= begin
+                          max_history = case 
+                                        when @ticks > 20
+                                          count = @ticks - @last_count
+                                          count = 1 if count == 0
+                                          if @max
+                                            times = @max / count
+                                            num = times / 20
+                                            num = 2 if num < 2
+                                          else
+                                            num = 10
+                                          end
+                                          count * num
+                                        else
+                                          20
+                                        end
+                          max_history = 30 if max_history > 30
+                          max_history
                         end
-                        count * num
-                      else
-                        20
-                      end
-        max_history = 30 if max_history > 30
         @history.shift if @history.length > max_history
       end
 
       @last_ticks = @ticks
 
       @mean_max ||= 0
-      if @history.length > 2
+      if @history.length > 3
 
         sticks, stime = @history.first
         ssticks, sstime = @history[-3]

data/lib/rbbt/util/log/progress/util.rb

@@ -80,10 +80,11 @@ module Log
 
     def self.with_bar(max, options = {})
       bar = new_bar(max, options)
+      res = nil
       begin
         error = false
-        yield bar
         keep = false
+        yield bar
       rescue KeepBar
         keep = true
       rescue

data/lib/rbbt/util/misc/omics.rb

@@ -511,9 +511,9 @@ module Misc
 
            case chr_prefix.to_s.downcase
            when "remove"
-             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
-           when "true", "add"
              sort_order = sort_order.collect{|chr| chr.sub('chr', '') }  if sort_order.first.include?('chr') 
+           when "true", "add"
+             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
            end
 
            sort_genomic_locations_by_contig(mutations, sort_order)

data/lib/rbbt/util/misc/system.rb

@@ -112,10 +112,10 @@ end
     res
   end
 
-  def self.is_filename?(string)
+  def self.is_filename?(string, need_to_exists = true)
     return true if defined? Path and Path === string
     return true if string.respond_to? :exists
-    return true if String === string and string.length < 265 and File.exist?(string)
+    return true if String === string and string.length < 265 and (File.exist?(string) || ! need_to_exists)
     return false
   end
 

data/lib/rbbt/util/python.rb

@@ -3,10 +3,56 @@ require 'pycall/import'
 
 module RbbtPython
   extend PyCall::Import
+
+  def self.exec(script)
+    PyCall.exec(script)
+  end
+
+  def self.iterate(iterator, options = {})
+    bar = options[:bar]
+
+    case bar
+    when TrueClass
+      bar = Log::ProgressBar.new nil, :desc => "RbbtPython iterate"
+    when String
+      bar = Log::ProgressBar.new nil, :desc => bar
+    end
+
+    while true
+      begin
+        yield iterator.__next__
+        bar.tick if bar
+      rescue PyCall::PyError
+        if $!.type.to_s == "<class 'StopIteration'>"
+          break
+        else
+          raise $!
+        end
+      rescue
+        bar.error if bar
+        raise $!
+      end
+    end
+
+    Log::ProgressBar.remove_bar bar if bar
+    nil
+  end
+
+  def self.collect(iterator, options = {}, &block)
+    acc = []
+    self.iterate(iterator, options) do |elem|
+      res = block.call elem
+      acc << res
+    end
+    acc
+  end
+
   def self.run(mod = nil, imports = nil, &block)
     if mod
-      if imports
+      if Array === imports
         pyfrom mod, :import => imports
+      elsif Hash === imports
+        pyimport mod, imports
       else
         pyimport mod 
       end
@@ -17,8 +63,10 @@ module RbbtPython
 
   def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
     if mod
-      if imports
+      if Array === imports
         pyfrom mod, :import => imports
+      elsif Hash === imports
+        pyimport mod, imports
       else
         pyimport mod 
       end
@@ -31,8 +79,10 @@ module RbbtPython
 
   def self.run_log_stderr(mod = nil, imports = nil, severity = 0, &block)
     if mod
-      if imports
+      if Array === imports
         pyfrom mod, :import => imports
+      elsif Hash === imports
+        pyimport mod, imports
       else
         pyimport mod 
       end
@@ -49,4 +99,14 @@ module RbbtPython
     end
   end
 
+  def self.add_paths(paths)
+    self.run 'sys' do
+      paths.each do |path|
+        sys.path.append path
+      end
+    end
+  end
+
+  RbbtPython.add_paths Rbbt.python.find_all
+  RbbtPython.pyimport "rbbt"
 end

data/lib/rbbt/util/simpleDSL.rb

@@ -38,13 +38,13 @@ module SimpleDSL
     hook_method(method)
 
     # Execute
+    @config ||= {}
     if actions.is_a? Proc
       begin
-        require 'parse_tree_extensions'
-        require 'parse_tree'
-        require 'ruby2ruby'
-        @config[@@method_name] = actions.to_ruby.collect[1..-2].join
+        require 'method_source'
+        @config[@@method_name] = actions.source.split("\n")[1..-2] * "\n"
       rescue Exception
+        Log.exception $!
         @config[@@method_name] = NoRuby2Ruby.new "The gem ruby2ruby is not installed. It will not work on ruby 1.9."
       end
       

data/lib/rbbt/workflow.rb

@@ -347,6 +347,11 @@ module Workflow
 
     inputs = IndiferentHash.setup(inputs)
 
+    not_overriden = inputs.delete :not_overriden 
+    if not_overriden
+      inputs[:not_overriden] = :not_overriden_dep
+    end
+
     Workflow.resolve_locals(inputs)
 
     task_info = task_info(taskname)
@@ -400,20 +405,43 @@ module Workflow
     jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
 
     dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
-    overriden = has_overriden_inputs || dependencies.select{|dep| dep.overriden }.any?
+
+    overriden_deps = dependencies.select{|d| d.overriden }
+    true_overriden_deps = overriden_deps.select{|d| TrueClass === d.overriden }
+
+    overriden = has_overriden_inputs || overriden_deps.any?
+
+    extension = task.extension
+
+    if extension == :dep_task
+      extension = nil
+      if dependencies.any?
+        dep_basename = File.basename(dependencies.last.path)
+        extension = dep_path.split(".").last if dep_basename.include?('.')
+      end
+    end
 
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
-      step_path = step_path taskname, jobname, [], [], task.extension
+      step_path = step_path taskname, jobname, [], [], extension
       input_values = task.take_input_values(inputs)
     else
       input_values = task.take_input_values(inputs)
-      step_path = step_path taskname, jobname, input_values, dependencies, task.extension
+      step_path = step_path taskname, jobname, input_values, dependencies, extension
     end
 
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
-    job.overriden = overriden
+
+    case not_overriden 
+    when TrueClass
+      job.overriden = has_overriden_inputs || true_overriden_deps.any?
+    when :not_overriden_dep
+      job.overriden = true if has_overriden_inputs || true_overriden_deps.any?
+    else
+      job.overriden = true if has_overriden_inputs || overriden_deps.any?
+    end
+
     job.real_inputs = real_inputs.keys
     job
   end

data/lib/rbbt/workflow/definition.rb

@@ -73,7 +73,7 @@ module Workflow
   REMOVE_DEP_TASKS = ENV["RBBT_REMOVE_DEP_TASKS"] == "true"
   def dep_task(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
-    extension workflow.tasks[oname].extension if workflow.tasks.include?(oname) unless @extension
+    extension :dep_task unless @extension
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?

data/lib/rbbt/workflow/step.rb

@@ -34,13 +34,40 @@ class Step
                     end
   end
 
+
+  def overriden?
+    return true if @overriden
+    return true if dependencies.select{|dep| dep.overriden? }.any?
+    info[:archived_info].each do |f,i|
+      return true if i[:overriden] || i["overriden"]
+    end if info[:archived_info]
+    return false
+  end
+
   def overriden
-    if @overriden.nil? 
-      return [] if dependencies.nil?
-      dependencies.select{|dep| dep.overriden }.any?
-    else
-      @overriden
+    @overriden
+    #if @overriden.nil? 
+    #  return false if dependencies.nil?
+    #  dependencies.select{|dep| dep.overriden? }.any?
+    #else
+    #  @overriden
+    #end
+  end
+
+  def overriden_deps
+    ord = []
+    deps = dependencies.dup
+    while dep = deps.shift
+      case dep.overriden
+      when FalseClass
+        next
+      when Symbol
+        ord << dep
+      else
+        deps += dep.dependencies
+      end
     end
+    ord
   end
 
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
@@ -134,7 +161,11 @@ class Step
 
     archived_info = {}
     dependencies.each do |dep|
-      archived_info[dep.path] = dep.info
+      if Symbol === dep.overriden && ! Open.exists?(dep.info_file)
+        archived_info[dep.path] = dep.overriden
+      else
+        archived_info[dep.path] = dep.info
+      end
       archived_info.merge!(dep.archived_info)
     end if dependencies
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -248,7 +248,7 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)
@@ -551,7 +551,7 @@ class Step
 
   def aborted?
     status = self.status
-    status == :aborted || ((status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
+    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
   end
 
   # {{{ INFO