rbbt-util 5.32.13 → 5.32.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c61ee9aa98e617b2cd43ea3c57b71439cbb5c2c65282ed7695fafd5c46ccbf3
-  data.tar.gz: e30e12b162f86ee74b4a4c33e44c7e976612a11e8219b2afd9c6cc0ea520fe24
+  metadata.gz: 54a6869616f33a2c526e35ddddf90c1e7c29e79c5df9fabaa51d9711e0aba249
+  data.tar.gz: 7a6c7290ff27c00ba5911a609897eb9104b357adec7a38b892ad5a8c01267e26
 SHA512:
-  metadata.gz: abee1a9322982a9da1cca62f7c36e47a7517a6c8066177f5155f8bfc17e6bc19054bfc1b8068c67e58fb0bf4a1c7585c3d493c2ca70d600391348e03f61ebab6
-  data.tar.gz: 80e3f811802c5e33cefd1c7c9561a9bdfcc2cfbfa97f300babd3c095de1d5bfb14a88e2e52918c4be9e869f9bf2722df68a493895077101e42b2f4ee57e9427c
+  metadata.gz: 1d7f69b1680ffc9cc8d30435da870ed50411d02ac285ab8b98f98f4fd8f9b3e07bf3b228904f5dc5ea03a987e4b37de398e99d50e8679f833682a0ecba8f632a
+  data.tar.gz: 35ff2212eccbb3fbb2141f7aa7ec1c6d6de4ae77f3e2289ce8d99b11ff5d657d967a25367e5710e0f1fa13f0e7cff2a6f525b13ce68b82b933a7113b16146b10

data/lib/rbbt/annotations/util.rb

@@ -152,6 +152,7 @@ module Annotated
       tsv = TSV.setup({}, :key_field => "List", :fields => fields, :type => :list, :unnamed => true)
 
       annot_id = annotations.id
+      annot_id = annot_id * "," if Array === annot_id
       tsv[annot_id] = annotations.tsv_values(*fields).dup
 
     when Array === annotations 

data/lib/rbbt/entity.rb

@@ -197,7 +197,7 @@ module Entity
             if self.instance_variable_get("@multiple_result_" + name.to_s)
               return self.instance_variable_get("@multiple_result_" + name.to_s)
             end
-            raise MultipleEntity, "Entity #{name} runs with multiple entities"
+            raise MultipleEntity, "Entity property #{name} runs with multiple entities"
           end
 
           define_method name do |*args|
@@ -225,10 +225,12 @@ module Entity
             case res
             when Array
               missing.zip(res).each do |o,res|
+                raise "Multiple function #{name} result nil for element #{o}" if res.nil?
                 o.instance_variable_set("@multiple_result_" + name.to_s, res)
               end
             when Hash
               res.each do |o,res|
+                raise "Multiple function #{name} result nil for element #{o}" if res.nil?
                 o.instance_variable_set("@multiple_result_" + name.to_s, res)
               end
             end
@@ -254,7 +256,10 @@ module Entity
 
         orig_method_name = method_name
         multi_name = "_multiple_" + method_name.to_s
+        single_name = "_single_" + method_name.to_s
+
         method_name = multi_name if self.instance_methods.include?(multi_name.to_sym)
+        method_name = single_name if self.instance_methods.include?(single_name.to_sym)
 
         orig_name = UNPERSISTED_PREFIX + method_name.to_s
         alias_method orig_name, method_name unless self.instance_methods.include? orig_name.to_sym

data/lib/rbbt/hpc/batch.rb

@@ -139,7 +139,7 @@ EOF
 
       keys = [
         :batch_dir,
-        :batch_modules,
+        :lua_modules,
         :batch_name,
         :contain,
         :contain_and_sync,
@@ -157,6 +157,7 @@ EOF
         :singularity_ruby_inline,
         :sync,
         :task_cpus,
+        :gres,
         :mem,
         :mem_per_cpu,
         :licenses,
@@ -289,7 +290,7 @@ let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
     end
 
     def prepare_environment(options = {})
-      modules = options[:batch_modules]
+      modules = options[:lua_modules]
 
       prepare_environment = ""
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -97,6 +97,7 @@ module HPC
         chains[job] ||= [] 
         chains[job] << dep 
         chains[job].concat chains[dep] if chains[dep]
+        chains[job].uniq!
       end
 
       chains

data/lib/rbbt/hpc/slurm.rb

@@ -30,6 +30,7 @@ export BATCH_SYSTEM=SLURM
       highmem        = Misc.process_options options, :highmem
       licenses       = Misc.process_options options, :licenses
       constraint     = Misc.process_options options, :constraint
+      gres           = Misc.process_options options, :gres
 
       mem            = Misc.process_options options, :mem
       mem_per_cpu    = Misc.process_options options, :mem_per_cpu
@@ -50,6 +51,7 @@ export BATCH_SYSTEM=SLURM
                        "time" => time,
                        "exclusive" => exclusive,
                        "licenses" => licenses,
+                       "gres" => gres,
                        "mem" => mem,
                        "mem-per-cpu" => mem_per_cpu,
       }

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -97,12 +97,11 @@ module TSV
         end
       end
       rescue
-        Log.exception $!
         error = true
         raise $!
       ensure
         join.call(error) if join
-        Log::ProgressBar.remove_bar(bar) if bar
+        Log::ProgressBar.remove_bar(bar, error) if bar
       end
   end
 
@@ -138,7 +137,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -178,7 +177,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -220,7 +219,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -274,7 +273,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -319,7 +318,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -491,9 +490,7 @@ module TSV
       q.join
       raise $!
     ensure
-      if bar
-        Log::ProgressBar.remove_bar(bar, error)
-      end
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 

data/lib/rbbt/util/log/progress/util.rb

@@ -80,10 +80,11 @@ module Log
 
     def self.with_bar(max, options = {})
       bar = new_bar(max, options)
+      res = nil
       begin
         error = false
-        yield bar
         keep = false
+        yield bar
       rescue KeepBar
         keep = true
       rescue

data/lib/rbbt/util/misc/omics.rb

@@ -511,9 +511,9 @@ module Misc
 
            case chr_prefix.to_s.downcase
            when "remove"
-             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
-           when "true", "add"
              sort_order = sort_order.collect{|chr| chr.sub('chr', '') }  if sort_order.first.include?('chr') 
+           when "true", "add"
+             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
            end
 
            sort_genomic_locations_by_contig(mutations, sort_order)

data/lib/rbbt/util/misc/system.rb

@@ -112,10 +112,10 @@ end
     res
   end
 
-  def self.is_filename?(string)
+  def self.is_filename?(string, need_to_exists = true)
     return true if defined? Path and Path === string
     return true if string.respond_to? :exists
-    return true if String === string and string.length < 265 and File.exist?(string)
+    return true if String === string and string.length < 265 and (File.exist?(string) || ! need_to_exists)
     return false
   end
 

data/lib/rbbt/util/python.rb

@@ -8,6 +8,45 @@ module RbbtPython
     PyCall.exec(script)
   end
 
+  def self.iterate(iterator, options = {})
+    bar = options[:bar]
+
+    case bar
+    when TrueClass
+      bar = Log::ProgressBar.new nil, :desc => "RbbtPython iterate"
+    when String
+      bar = Log::ProgressBar.new nil, :desc => bar
+    end
+
+    while true
+      begin
+        yield iterator.__next__
+        bar.tick if bar
+      rescue PyCall::PyError
+        if $!.type.to_s == "<class 'StopIteration'>"
+          break
+        else
+          raise $!
+        end
+      rescue
+        bar.error if bar
+        raise $!
+      end
+    end
+
+    Log::ProgressBar.remove_bar bar if bar
+    nil
+  end
+
+  def self.collect(iterator, options = {}, &block)
+    acc = []
+    self.iterate(iterator, options) do |elem|
+      res = block.call elem
+      acc << res
+    end
+    acc
+  end
+
   def self.run(mod = nil, imports = nil, &block)
     if mod
       if Array === imports

data/lib/rbbt/workflow/util/data.rb

@@ -23,7 +23,11 @@ module Workflow
         task_info = workflow.task_info(oname)
         dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
         data_options = block.call data_dir, dir_options, task_info
-        {:inputs => data_options.merge(options)}
+        if data_options.include?(:inputs)
+          data_options
+        else
+          {:inputs => data_options.merge(options)}
+        end
       end
     end
   end

data/python/rbbt.py

@@ -1,3 +1,7 @@
+import warnings
+import sys
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
 def rbbt():
     print("Rbbt")
-

data/share/rbbt_commands/hpc/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/hpc/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/lsf/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/lsf/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/slurm/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/slurm/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/test/rbbt/workflow/util/test_data.rb

@@ -5,15 +5,24 @@ module TestDataWF
   extend Workflow
   extend Workflow::Data
 
+  input :salutation, :string
+  task :salute_luis => :string do |name,salutation|
+    "Hi Luis: #{salutation}"
+  end
+
   input :name, :string
   input :salutation, :string
   task :salute => :string do |name,salutation|
     "Hi #{name}: #{salutation}"
   end
 
-  data_task :salute_data, TestDataWF, :salute do |directory,options|
+  data_task :salute_data, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
     options.merge({:salutation => directory.salutation.read})
   end
+
+  data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
+    {:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
+  end
 end
 
 class TestWorkflowData < Test::Unit::TestCase
@@ -28,7 +37,11 @@ class TestWorkflowData < Test::Unit::TestCase
 
       job = TestDataWF.job(:salute_data, "TestDir")
       job.recursive_clean.run
-      ppp job.run
+      assert job.run.include? "Miguel"
+
+      job = TestDataWF.job(:salute_data2, "TestDir")
+      job.recursive_clean.run
+      assert job.run.include? "Luis"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.13
+  version: 5.32.15
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-26 00:00:00.000000000 Z
+date: 2021-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake