rbbt-util 5.33.1 → 5.33.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70ba4440be0bdca6f80b4aa78393ca4fbec5dbc8d8a1cc72b6393e73127b5340
-  data.tar.gz: 42021bd242233b5e4e3a268c27291953d84c8bad85b87d8feaf6389685f043bf
+  metadata.gz: 1e780efeaa15611ce5835bb314b9ad378719f9f744228a2c9a8a765c202e10e1
+  data.tar.gz: 7338f595861b6f93ea0043ffabe9e78762c1acc478669f989ea23dc84820a213
 SHA512:
-  metadata.gz: d6a229aa39440cf3b0b42a505e32c699d791d8dd1e1328f42cd6500c947560b0d9343312080af78475a6f4835077e53c029d862e9d4c90addd84408925754b50
-  data.tar.gz: e31505e92ca6425095a90bdcd8fe95d435c8435bc929dcb1c0c77083a1286439c952d7a98af1091664fcee7a57ba3f86b99258a84958714d630ab4a3678f1ee3
+  metadata.gz: 6650b5f00f6e2b8a9ccb3b2d389357ee0b9f551166d185aa2c5ec2c635150e2274c75af35fa6dc66cb09abeaa3c9e445a344015bee616ef78607933337c67e6f
+  data.tar.gz: 4847e30f91c1fc388cb83bb842966fdf73cf150d4ed528b89e755d8ae5ed732e6886b7939031664fa0671b386c34e3daaf4ce0873f0991c4711b70bce03ed5c2

data/lib/rbbt/hpc/batch.rb

@@ -538,10 +538,13 @@ env > #{batch_options[:fenv]}
       workflow = job.workflow
       task_name = job.task_name
 
+      workflows_to_load = job.rec_dependencies.select{|d| Step === d}.collect{|d| d.workflow }.compact.collect(&:to_s) - [workflow.to_s]
+
       TmpFile.with_file(nil, remove_batch_dir, :tmpdir => batch_base_dir, :prefix => "#{system}_rbbt_job-#{workflow.to_s}-#{task_name}-") do |batch_dir|
         Misc.add_defaults options, 
           :batch_dir => batch_dir, 
-          :inputs_dir => File.join(batch_dir, "inputs_dir")
+          :inputs_dir => File.join(batch_dir, "inputs_dir"),
+          :workflows => workflows_to_load * ","
 
         options[:procpath_performance] ||= File.join(batch_dir, "procpath##{procpath.gsub(',', '#')}") if procpath
 

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -11,7 +11,7 @@ module HPC
         iii :END_BATCHES
       else
         n = batch.dup
-        n[:deps] = n[:deps].collect{|b| b[:top_level] }
+        n[:deps] = n[:deps].collect{|b| b[:top_level] } if n[:deps]
         iif n
       end
     end
@@ -79,14 +79,14 @@ module HPC
 
       batches.each do |batch|
         jobs = batch[:jobs]
-        all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
+        all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq - jobs
 
         minimum = all_deps
         all_deps.each do |dep|
           minimum -= job_dependencies(dep)
         end
 
-        all_deps = minimum
+        all_deps = minimum 
         deps = all_deps.collect do |d|
           (batches - [batch]).select{|batch| batch[:jobs].collect(&:path).include? d.path }
         end.flatten.uniq

data/lib/rbbt/hpc/orchestrate.rb

@@ -14,8 +14,14 @@ module HPC
       options.delete "detach"
       options.delete "jobname"
 
-      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
-      rules ||= {}
+      if options[:orchestration_rules]
+        rules = YAML.load(Open.read(options[:orchestration_rules]))
+      elsif Rbbt.etc.slurm["default.yaml"].exists?
+        rules = YAML.load(Open.read(Rbbt.etc.slurm["default.yaml"]))
+      else
+        rules = {}
+      end
+
       IndiferentHash.setup(rules)
 
       batches = HPC::Orchestration.job_batches(rules, job)

data/lib/rbbt/resource.rb

@@ -270,6 +270,7 @@ source "$INSTALL_HELPER_FILE"
                               git = content[:git]
                               src = content[:src]
                               url = content[:url]
+                              jar = content[:jar]
                               extra = content[:extra]
                               commands = content[:commands]
                               if git
@@ -290,6 +291,16 @@ url='#{src}'
 
 install_src "$name" "$url" #{extra}
 
+#{commands}
+                                EOF
+                              elsif jar
+                                <<-EOF
+
+name='#{name}'
+url='#{jar}'
+
+install_jar "$name" "$url" #{extra}
+
 #{commands}
                                 EOF
                               else

data/lib/rbbt/util/cmd.rb

@@ -75,7 +75,7 @@ module CMD
 
     string = ""
     options.each do |option, value|
-      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=]+$/i
+      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
       #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
       value = value.gsub("'","\\'") if value.to_s.include? "'"
 

data/lib/rbbt/workflow/definition.rb

@@ -92,7 +92,12 @@ module Workflow
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
-        Open.ln_h dep.path, self.tmp_path
+
+        if dep.overriden
+          Open.link dep.path, self.tmp_path
+        else
+          Open.ln_h dep.path, self.tmp_path
+        end
 
         case remove.to_s
         when 'true'

data/lib/rbbt/workflow/integration/nextflow.rb

@@ -4,7 +4,7 @@ module Workflow
   end
 
   def self.nextflow_includes(file)
-    Open.read(file).scan(/^include\s*{\s*(.*?)(?:\s*as.*?)?}\s*from\s+"(.*?)"(?:\s*params.*)?/).collect{|p| p}.uniq
+    Open.read(file).scan(/^include\s*{\s*([^\s]*?)\s+.*?}\s*from\s+["'](.*?)["'](?:\s*params.*)?/).collect{|p| p}.uniq
   end
 
   def self.nextflow_recursive_params(file)
@@ -48,25 +48,43 @@ module Workflow
     task name => result do 
       work = file('work')
       profile = config :profile, :nextflow
+      config_file = config :config, :nextflow
 
-      new_inputs = inputs.zip(inputs.fields).collect do |v,f|
-        if String === v && m = v.match(/^JOB_FILE:(.*)/)
-          file(m[1]) 
-        elsif v.nil?
-          Rbbt::Config.get(['nextflow', f] * "_", 'default', f)
+      nextflow_inputs = {}
+
+      inputs.zip(inputs.fields).collect do |v,f|
+        v = if String === v && m = v.match(/^JOB_FILE:(.*)/)
+              file(m[1]) 
+            elsif v.nil?
+              Rbbt::Config.get(['nextflow', f] * "_", 'default', f)
+            else
+              v
+            end
+
+        if f.to_s.include?("-") 
+          p,_sep, section = f.to_s.partition("-")
+          name = [section, p] * "."
         else
-          v
+          name = f
         end
+          
+        nextflow_inputs[name] = v
       end
       
-      inputs.replace new_inputs
-
       Misc.in_dir file('stage') do
-        if profile
-          cmd("nextflow run -work-dir #{work} -ansi-log false  -profile #{profile} #{file}", inputs.to_hash.merge('add_option_dashes' => true))
-        else
-          cmd("nextflow run -work-dir #{work} -ansi-log false #{file}", inputs.to_hash.merge('add_option_dashes' => true))
-        end
+
+        cmd = "nextflow "
+
+        cmd += " -C #{config_file}" if config_file
+
+        cmd += " run"
+
+        cmd += " -work-dir #{work} -ansi-log false"
+
+        cmd += " -profile #{profile}" if profile
+
+
+        cmd("#{cmd} #{file}", nextflow_inputs.merge('add_option_dashes' => true))
       end
 
       output_file = file(output).glob.first if output

data/lib/rbbt/workflow/step/dependencies.rb

@@ -512,6 +512,7 @@ class Step
     return true if @overriden
     return true if dependencies.select{|dep| dep.overriden? }.any?
     info[:archived_info].each do |f,i|
+      next if Symbol === i
       return true if i[:overriden] || i["overriden"]
     end if info[:archived_info]
     return false

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -84,7 +84,8 @@ module Workflow
 
     def self.candidates(workload, rules)
       if rules.empty?
-        candidates = workload.select{|k,v| v.empty? }.
+        candidates = workload.
+          select{|k,v| v.empty? }.
           collect{|k,v| k }.
           reject{|k| k.done? }
       else
@@ -204,7 +205,6 @@ module Workflow
             when (job.error? || job.aborted?)
               begin
                 if job.recoverable_error?
-                  iif [:CLEAN, job, job.status, job.info[:exception]]
                   job.clean
                   raise TryAgain
                 else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.1
+  version: 5.33.4
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-22 00:00:00.000000000 Z
+date: 2022-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake