rbbt-util 5.32.13 → 5.32.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c61ee9aa98e617b2cd43ea3c57b71439cbb5c2c65282ed7695fafd5c46ccbf3
-  data.tar.gz: e30e12b162f86ee74b4a4c33e44c7e976612a11e8219b2afd9c6cc0ea520fe24
+  metadata.gz: adee22216a6bbce0a4292f20e89f3c620fc47fcb28aaaf15082cd3ee54d2c093
+  data.tar.gz: 920f61f95dce1acbecc279ae09731dab297d0eb57cffe7a08734297e917436f8
 SHA512:
-  metadata.gz: abee1a9322982a9da1cca62f7c36e47a7517a6c8066177f5155f8bfc17e6bc19054bfc1b8068c67e58fb0bf4a1c7585c3d493c2ca70d600391348e03f61ebab6
-  data.tar.gz: 80e3f811802c5e33cefd1c7c9561a9bdfcc2cfbfa97f300babd3c095de1d5bfb14a88e2e52918c4be9e869f9bf2722df68a493895077101e42b2f4ee57e9427c
+  metadata.gz: f7fb0024e458db313ab31a46c4142092121d76efc5ca79c85876a8dc412e6483c12cf3058a1f9f5e6300a31ce672e87b60864eff46c2af8662df411e5680ac39
+  data.tar.gz: 4216a058f4329d25ed573129c0b40408921566218267922f1b3831108ab2745fc36279fdf7ecedd580d71c0163fae44a8d36adfc2223fecad76ac5864ce90c5f

data/lib/rbbt/annotations/util.rb

@@ -152,6 +152,7 @@ module Annotated
       tsv = TSV.setup({}, :key_field => "List", :fields => fields, :type => :list, :unnamed => true)
 
       annot_id = annotations.id
+      annot_id = annot_id * "," if Array === annot_id
       tsv[annot_id] = annotations.tsv_values(*fields).dup
 
     when Array === annotations 

data/lib/rbbt/entity.rb

@@ -197,7 +197,7 @@ module Entity
             if self.instance_variable_get("@multiple_result_" + name.to_s)
               return self.instance_variable_get("@multiple_result_" + name.to_s)
             end
-            raise MultipleEntity, "Entity #{name} runs with multiple entities"
+            raise MultipleEntity, "Entity property #{name} runs with multiple entities"
           end
 
           define_method name do |*args|
@@ -225,10 +225,12 @@ module Entity
             case res
             when Array
               missing.zip(res).each do |o,res|
+                raise "Multiple function #{name} result nil for element #{o}" if res.nil?
                 o.instance_variable_set("@multiple_result_" + name.to_s, res)
               end
             when Hash
               res.each do |o,res|
+                raise "Multiple function #{name} result nil for element #{o}" if res.nil?
                 o.instance_variable_set("@multiple_result_" + name.to_s, res)
               end
             end
@@ -254,7 +256,10 @@ module Entity
 
         orig_method_name = method_name
         multi_name = "_multiple_" + method_name.to_s
+        single_name = "_single_" + method_name.to_s
+
         method_name = multi_name if self.instance_methods.include?(multi_name.to_sym)
+        method_name = single_name if self.instance_methods.include?(single_name.to_sym)
 
         orig_name = UNPERSISTED_PREFIX + method_name.to_s
         alias_method orig_name, method_name unless self.instance_methods.include? orig_name.to_sym

data/lib/rbbt/hpc/batch.rb

@@ -139,7 +139,7 @@ EOF
 
       keys = [
         :batch_dir,
-        :batch_modules,
+        :lua_modules,
         :batch_name,
         :contain,
         :contain_and_sync,
@@ -157,6 +157,7 @@ EOF
         :singularity_ruby_inline,
         :sync,
         :task_cpus,
+        :gres,
         :mem,
         :mem_per_cpu,
         :licenses,
@@ -289,7 +290,7 @@ let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
     end
 
     def prepare_environment(options = {})
-      modules = options[:batch_modules]
+      modules = options[:lua_modules]
 
       prepare_environment = ""
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -97,6 +97,7 @@ module HPC
         chains[job] ||= [] 
         chains[job] << dep 
         chains[job].concat chains[dep] if chains[dep]
+        chains[job].uniq!
       end
 
       chains

data/lib/rbbt/hpc/slurm.rb

@@ -30,6 +30,7 @@ export BATCH_SYSTEM=SLURM
       highmem        = Misc.process_options options, :highmem
       licenses       = Misc.process_options options, :licenses
       constraint     = Misc.process_options options, :constraint
+      gres           = Misc.process_options options, :gres
 
       mem            = Misc.process_options options, :mem
       mem_per_cpu    = Misc.process_options options, :mem_per_cpu
@@ -50,6 +51,7 @@ export BATCH_SYSTEM=SLURM
                        "time" => time,
                        "exclusive" => exclusive,
                        "licenses" => licenses,
+                       "gres" => gres,
                        "mem" => mem,
                        "mem-per-cpu" => mem_per_cpu,
       }

data/lib/rbbt/resource/path.rb

@@ -159,6 +159,12 @@ module Path
   end
 
   def find(where = nil, caller_lib = nil, paths = nil)
+
+    if located?
+      self.original ||= self
+      return self
+    end
+
     @path ||= {}
     rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil 
     key_elems = [where, caller_lib, rsearch_paths, paths]
@@ -167,12 +173,6 @@ module Path
 
     return @path[key] if @path[key]
 
-    if located?
-      @path[key] = self
-      self.original ||= self
-      return self
-    end
-
     @path[key] ||= begin
                      paths = [paths, rsearch_paths, self.search_paths, SEARCH_PATHS].reverse.compact.inject({}){|acc,h| acc.merge! h; acc }
                      where = paths[:default] if where == :default

data/lib/rbbt/tsv.rb

@@ -98,6 +98,11 @@ module TSV
         stream = get_stream source, options.merge(open_options)
         parse stream, data, options.merge(:tsv_grep => tsv_grep)
 
+        if ! open_options[:noclose]
+          stream.close unless stream.closed?
+          stream.join if stream.respond_to?(:join)
+        end
+
         data.filename = filename.to_s unless filename.nil?
 
         if data.identifiers.nil? and Path === filename and filename.identifier_file_path

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -97,12 +97,11 @@ module TSV
         end
       end
       rescue
-        Log.exception $!
         error = true
         raise $!
       ensure
         join.call(error) if join
-        Log::ProgressBar.remove_bar(bar) if bar
+        Log::ProgressBar.remove_bar(bar, error) if bar
       end
   end
 
@@ -138,7 +137,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -178,7 +177,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -220,7 +219,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -274,7 +273,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -319,7 +318,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -333,7 +332,7 @@ module TSV
     Log.low{"Traversing #{name} #{Log.color :green, "->"} #{stream_name(options[:into])}"}
     begin
       case obj
-      when (defined? FastContainers and FastContainers::PriorityQueue)
+      when (defined? FastContainers && FastContainers::PriorityQueue === obj)
         traverse_priority_queue(obj, options, &block)
       when TSV
         traverse_tsv(obj, options, &block)
@@ -390,7 +389,7 @@ module TSV
       when Set
         traverse_array(obj.to_a, options, &block)
       when String
-        if Open.remote? obj or Misc.is_filename? obj
+        if Open.remote?(obj) or Misc.is_filename?(obj)
           Open.open(obj) do |s|
             traverse_obj(s, options, &block)
           end
@@ -491,9 +490,7 @@ module TSV
       q.join
       raise $!
     ensure
-      if bar
-        Log::ProgressBar.remove_bar(bar, error)
-      end
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 

data/lib/rbbt/util/log.rb

@@ -322,8 +322,13 @@ module Log
 
   def self.exception(e)
     stack = caller
-    error([e.class.to_s, e.message].compact * ": " )
-    error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
+    if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+      error([e.class.to_s, e.message].compact * ": " )
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
+    else
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
+      error([e.class.to_s, e.message].compact * ": " )
+    end
   end
 
   def self.deprecated(m)
@@ -370,11 +375,17 @@ module Log
 
   def self.stack(stack)
     LOG_MUTEX.synchronize do
-
-      STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
+      if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+        STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
       color_stack(stack).each do |line|
         STDERR.puts line
       end
+      else
+        STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
+        color_stack(stack.reverse).each do |line|
+          STDERR.puts line
+        end
+      end
     end
   end
 

data/lib/rbbt/util/log/progress/util.rb

@@ -80,10 +80,11 @@ module Log
 
     def self.with_bar(max, options = {})
       bar = new_bar(max, options)
+      res = nil
       begin
         error = false
-        yield bar
         keep = false
+        yield bar
       rescue KeepBar
         keep = true
       rescue

data/lib/rbbt/util/misc/inspect.rb

@@ -320,7 +320,9 @@ module Misc
               end
             end
           when String
-            if Misc.is_filename?(obj) and ! %w(. ..).include?(obj)
+            good_filename = Misc.is_filename?(obj, false) && ! %w(. ..).include?(obj) && %w(. /).include?(obj[0])
+            is_path = Path === obj
+            if good_filename && ! is_path
               obj2str Path.setup(obj.dup)
             else
               obj = obj.chomp if String === obj

data/lib/rbbt/util/misc/omics.rb

@@ -511,9 +511,9 @@ module Misc
 
            case chr_prefix.to_s.downcase
            when "remove"
-             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
-           when "true", "add"
              sort_order = sort_order.collect{|chr| chr.sub('chr', '') }  if sort_order.first.include?('chr') 
+           when "true", "add"
+             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
            end
 
            sort_genomic_locations_by_contig(mutations, sort_order)

data/lib/rbbt/util/misc/system.rb

@@ -112,10 +112,10 @@ end
     res
   end
 
-  def self.is_filename?(string)
+  def self.is_filename?(string, need_to_exists = true)
     return true if defined? Path and Path === string
     return true if string.respond_to? :exists
-    return true if String === string and string.length < 265 and File.exist?(string)
+    return true if String === string and string.length < 265 and (File.exist?(string) || ! need_to_exists)
     return false
   end
 

data/lib/rbbt/util/open.rb

@@ -240,6 +240,7 @@ module Open
   end
 
   def self.find_repo_dir(file)
+    return nil
     self.repository_dirs.each do |dir|
       dir = dir + '/' unless dir.chars[-1] == "/"
 

data/lib/rbbt/util/python.rb

@@ -8,6 +8,45 @@ module RbbtPython
     PyCall.exec(script)
   end
 
+  def self.iterate(iterator, options = {})
+    bar = options[:bar]
+
+    case bar
+    when TrueClass
+      bar = Log::ProgressBar.new nil, :desc => "RbbtPython iterate"
+    when String
+      bar = Log::ProgressBar.new nil, :desc => bar
+    end
+
+    while true
+      begin
+        yield iterator.__next__
+        bar.tick if bar
+      rescue PyCall::PyError
+        if $!.type.to_s == "<class 'StopIteration'>"
+          break
+        else
+          raise $!
+        end
+      rescue
+        bar.error if bar
+        raise $!
+      end
+    end
+
+    Log::ProgressBar.remove_bar bar if bar
+    nil
+  end
+
+  def self.collect(iterator, options = {}, &block)
+    acc = []
+    self.iterate(iterator, options) do |elem|
+      res = block.call elem
+      acc << res
+    end
+    acc
+  end
+
   def self.run(mod = nil, imports = nil, &block)
     if mod
       if Array === imports

data/lib/rbbt/workflow.rb

@@ -1,4 +1,5 @@
 require 'rbbt/workflow/definition'
+require 'rbbt/workflow/dependencies'
 require 'rbbt/workflow/task'
 require 'rbbt/workflow/step'
 require 'rbbt/workflow/accessor'
@@ -228,6 +229,47 @@ module Workflow
                   end
   end
 
+  TAG = ENV["RBBT_INPUT_JOBNAME"] == "true" ? :inputs : :hash
+  DEBUG_JOB_HASH = ENV["RBBT_DEBUG_JOB_HASH"] == 'true'
+  def step_path(taskname, jobname, inputs, dependencies, extension = nil)
+    raise "Jobname makes an invalid path: #{ jobname }" if jobname.include? '..'
+    if inputs.length > 0 or dependencies.any?
+      tagged_jobname = case TAG
+                       when :hash
+                         clean_inputs = Annotated.purge(inputs)
+                         clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (d.overriden? ? d.path : d.short_path) : d }
+                         key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
+                         key_str = Misc.obj2str(key_obj)
+                         hash_str = Misc.digest(key_str)
+                         Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}" if DEBUG_JOB_HASH
+                         jobname + '_' << hash_str
+                       when :inputs
+                         all_inputs = {}
+                         inputs.zip(self.task_info(taskname)[:inputs]) do |i,f|
+                           all_inputs[f] = i
+                         end
+                         dependencies.each do |dep|
+                           ri = dep.recursive_inputs
+                           ri.zip(ri.fields).each do |i,f|
+                             all_inputs[f] = i
+                           end
+                         end
+
+                         all_inputs.any? ? jobname + '_' << Misc.obj2str(all_inputs) : jobname
+                       else
+                         jobname
+                       end
+    else
+      tagged_jobname = jobname
+    end
+
+    if extension and not extension.empty?
+      tagged_jobname = tagged_jobname + ('.' << extension.to_s)
+    end
+
+    workdir[taskname][tagged_jobname].find
+  end
   def import_task(workflow, orig, new)
     orig_task = workflow.tasks[orig]
     new_task = orig_task.dup
@@ -411,12 +453,22 @@ module Workflow
 
     overriden = has_overriden_inputs || overriden_deps.any?
 
+    extension = task.extension
+
+    if extension == :dep_task
+      extension = nil
+      if dependencies.any?
+        dep_basename = File.basename(dependencies.last.path)
+        extension = dep_basename.split(".").last if dep_basename.include?('.')
+      end
+    end
+
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
-      step_path = step_path taskname, jobname, [], [], task.extension
+      step_path = step_path taskname, jobname, [], [], extension
       input_values = task.take_input_values(inputs)
     else
       input_values = task.take_input_values(inputs)
-      step_path = step_path taskname, jobname, input_values, dependencies, task.extension
+      step_path = step_path taskname, jobname, input_values, dependencies, extension
     end
 
     job = get_job_step step_path, task, input_values, dependencies