rbbt-util 5.33.0 → 5.33.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a6b198e57fe73c0e36e7aea6164bafd1a8a97d10b3a992f17d0ba6dc531a3c7
-  data.tar.gz: 5aa8b8753fc88d5de6fa8f6db31b9b71a57ee5dcc86380041582a4985984f749
+  metadata.gz: 3ffb05883cf1e7eda628395b50967322b42240ace100fb73a2f4c4a1ed5e8d27
+  data.tar.gz: 8b25c002f68b88dc0ba1f09c041258ee0ca15d4fc6f8dc3df5de37ea2473d059
 SHA512:
-  metadata.gz: a61b6e6671803d73b7993b2de9c6cf16d4f463ddd04f9378b54c77a00eb7292b4e817eea14cce81704fe3f2640d38ede202a3928d293ab1fe7e0e67fa87a35b0
-  data.tar.gz: d41b1477aab34bfc08610602a3e333a5701edb58d9fa10ecf24436659c4e996bb000ceccb20bbcba1f153ec4e888264ae47ab28689b9e2879601da0b7822e831
+  metadata.gz: 7b74e527336830eecb1cad52bb13cbcb335c734a18e9261d87cfba6da93422cf94a826e97d43c6ef1d37d122623408230d80a037a9a91000fa95a6cf9c818cf6
+  data.tar.gz: 8b8de5c12aac8dfae5f68018cb5aae4023b78993fa78aa7cf93cacc16eea4c2b959f75190ead6862502ecc75ae3ccd9cc462b1db26930943447a758f1f359cd9

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -11,7 +11,7 @@ module HPC
         iii :END_BATCHES
       else
         n = batch.dup
-        n[:deps] = n[:deps].collect{|b| b[:top_level] }
+        n[:deps] = n[:deps].collect{|b| b[:top_level] } if n[:deps]
         iif n
       end
     end
@@ -79,7 +79,14 @@ module HPC
 
       batches.each do |batch|
         jobs = batch[:jobs]
-        all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
+        all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq - jobs
+
+        minimum = all_deps
+        all_deps.each do |dep|
+          minimum -= job_dependencies(dep)
+        end
+
+        all_deps = minimum 
         deps = all_deps.collect do |d|
           (batches - [batch]).select{|batch| batch[:jobs].collect(&:path).include? d.path }
         end.flatten.uniq

data/lib/rbbt/hpc/orchestrate.rb

@@ -14,8 +14,14 @@ module HPC
       options.delete "detach"
       options.delete "jobname"
 
-      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
-      rules ||= {}
+      if options[:orchestration_rules]
+        rules = YAML.load(Open.read(options[:orchestration_rules]))
+      elsif Rbbt.etc.slurm["default.yaml"].exists?
+        rules = YAML.load(Open.read(Rbbt.etc.slurm["default.yaml"]))
+      else
+        rules = {}
+      end
+
       IndiferentHash.setup(rules)
 
       batches = HPC::Orchestration.job_batches(rules, job)
@@ -26,7 +32,29 @@ module HPC
         raise "No batch without unmet dependencies" if top.nil?
         batches.delete top
         job_options = options.merge(top[:rules])
-        job_options.merge!(:batch_dependencies => top[:deps].nil? ? [] : top[:deps].collect{|d| batch_ids[d] })
+
+        if top[:deps].nil?
+          batch_dependencies = [] 
+        else 
+          top_jobs = top[:jobs]
+
+          batch_dependencies = top[:deps].collect{|d| 
+            target = d[:top_level]
+            canfail = false
+
+            top_jobs.each do |job|
+              canfail = true if job.canfail_paths.include?(target.path)
+            end
+
+            if canfail
+              'canfail:' + batch_ids[d].to_s
+            else
+              batch_ids[d].to_s
+            end
+          }
+        end
+
+        job_options.merge!(:batch_dependencies => batch_dependencies )
         job_options.merge!(:manifest => top[:jobs].collect{|d| d.task_signature })
 
         if options[:dry_run]

data/lib/rbbt/util/cmd.rb

@@ -75,7 +75,7 @@ module CMD
 
     string = ""
     options.each do |option, value|
-      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=]+$/i
+      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
       #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
       value = value.gsub("'","\\'") if value.to_s.include? "'"
 

data/lib/rbbt/util/misc/exceptions.rb

@@ -32,8 +32,6 @@ class DependencyError < Aborted
   def initialize(msg)
     if defined? Step and Step === msg
       step = msg
-      workflow = step.path.split("/")[-3]
-      new_msg = [workflow, step.short_path, step.messages.last] * " - "
       new_msg = [step.path, step.messages.last] * ": "
       super(new_msg)
     else
@@ -42,6 +40,21 @@ class DependencyError < Aborted
   end
 end
 
+class DependencyRbbtException < RbbtException
+  def initialize(msg)
+    if defined? Step and Step === msg
+      step = msg
+
+      new_msg = nil
+      new_msg = [step.path, step.messages.last] * ": "
+
+      super(new_msg)
+    else
+      super(msg)
+    end
+  end
+end
+
 class DontClose < Exception; end
 
 class KeepLocked < Exception
@@ -64,4 +77,3 @@ class StopInsist < Exception
     @exception = exception
   end
 end
-

data/lib/rbbt/workflow/definition.rb

@@ -92,7 +92,12 @@ module Workflow
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
-        Open.ln_h dep.path, self.tmp_path
+
+        if dep.overriden
+          Open.link dep.path, self.tmp_path
+        else
+          Open.ln_h dep.path, self.tmp_path
+        end
 
         case remove.to_s
         when 'true'

data/lib/rbbt/workflow/integration/nextflow.rb

@@ -4,7 +4,7 @@ module Workflow
   end
 
   def self.nextflow_includes(file)
-    Open.read(file).scan(/^include\s*{\s*(.*?)(?:\s*as.*?)?}\s*from\s+"(.*?)"(?:\s*params.*)?/).collect{|p| p}.uniq
+    Open.read(file).scan(/^include\s*{\s*([^\s]*?)\s+.*?}\s*from\s+["'](.*?)["'](?:\s*params.*)?/).collect{|p| p}.uniq
   end
 
   def self.nextflow_recursive_params(file)
@@ -48,25 +48,43 @@ module Workflow
     task name => result do 
       work = file('work')
       profile = config :profile, :nextflow
+      config_file = config :config, :nextflow
 
-      new_inputs = inputs.zip(inputs.fields).collect do |v,f|
-        if String === v && m = v.match(/^JOB_FILE:(.*)/)
-          file(m[1]) 
-        elsif v.nil?
-          Rbbt::Config.get(['nextflow', f] * "_", 'default', f)
+      nextflow_inputs = {}
+
+      inputs.zip(inputs.fields).collect do |v,f|
+        v = if String === v && m = v.match(/^JOB_FILE:(.*)/)
+              file(m[1]) 
+            elsif v.nil?
+              Rbbt::Config.get(['nextflow', f] * "_", 'default', f)
+            else
+              v
+            end
+
+        if f.to_s.include?("-") 
+          p,_sep, section = f.to_s.partition("-")
+          name = [section, p] * "."
         else
-          v
+          name = f
         end
+          
+        nextflow_inputs[name] = v
       end
       
-      inputs.replace new_inputs
-
       Misc.in_dir file('stage') do
-        if profile
-          cmd("nextflow run -work-dir #{work} -ansi-log false  -profile #{profile} #{file}", inputs.to_hash.merge('add_option_dashes' => true))
-        else
-          cmd("nextflow run -work-dir #{work} -ansi-log false #{file}", inputs.to_hash.merge('add_option_dashes' => true))
-        end
+
+        cmd = "nextflow "
+
+        cmd += " -C #{config_file}" if config_file
+
+        cmd += " run"
+
+        cmd += " -work-dir #{work} -ansi-log false"
+
+        cmd += " -profile #{profile}" if profile
+
+
+        cmd("#{cmd} #{file}", nextflow_inputs.merge('add_option_dashes' => true))
       end
 
       output_file = file(output).glob.first if output

data/lib/rbbt/workflow/step/dependencies.rb

@@ -107,7 +107,26 @@ class Step
       canfail = ComputeDependency === job && job.canfail?
     end
 
-    raise DependencyError, job if job.error? and not canfail
+    raise_dependency_error(job) if job.error? and not canfail
+  end
+
+  def self.raise_dependency_error(job)
+    begin
+      if job.get_exception
+        klass = job.get_exception.class
+      else
+        klass = Kernel.const_get(info[:exception][:class])
+      end
+    rescue
+      Log.exception $!
+      raise DependencyError, job 
+    end
+
+    if (klass <= RbbtException)
+      raise DependencyRbbtException, job 
+    else
+      raise DependencyError, job 
+    end
   end
 
   def log_dependency_exec(dependency, action)
@@ -169,7 +188,7 @@ class Step
 
       if dependency.error?
         log_dependency_exec(dependency, :error)
-        raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
+        raise_dependency_error dependency 
       end
 
       if dependency.streaming?
@@ -368,7 +387,7 @@ class Step
       seen_paths << step.path
       begin
         Step.prepare_for_execution(step) unless step == self
-      rescue DependencyError
+      rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
       next unless step.dependencies and step.dependencies.any?
@@ -493,6 +512,7 @@ class Step
     return true if @overriden
     return true if dependencies.select{|dep| dep.overriden? }.any?
     info[:archived_info].each do |f,i|
+      next if Symbol === i
       return true if i[:overriden] || i["overriden"]
     end if info[:archived_info]
     return false

data/lib/rbbt/workflow/step/info.rb

@@ -285,7 +285,7 @@ class Step
     begin
       return true unless info[:exception]
       klass = Kernel.const_get(info[:exception][:class])
-      ! (klass <= RbbtException)
+      ! (klass <= RbbtException )
     rescue Exception
       true
     end

data/lib/rbbt/workflow/step/run.rb

@@ -437,7 +437,7 @@ class Step
         end
       end # END SYNC
       res
-    rescue DependencyError
+    rescue DependencyError, DependencyRbbtException
       exception $!
     rescue LockInterrupted
       raise $!
@@ -484,7 +484,7 @@ class Step
     if dofork
       fork(true) unless started?
 
-      join unless done?
+      join unless done? or dofork == :nowait
     else
       run(true) unless started?
 
@@ -496,6 +496,7 @@ class Step
 
   def fork(no_load = false, semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid and Misc.pid_exists?(@pid) and not done? and info[:forked]
+    Log.debug "Fork to run #{self.path}"
     sout, sin = Misc.pipe if no_load == :stream
     @pid = Process.fork do
       Signal.trap(:TERM) do

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -71,6 +71,7 @@ module Workflow
 
       default_resources.each{|k,v| resources[k] ||= v } if default_resources
 
+      resources = {:cpus => 1} if resources.empty?
       resources
     end
 
@@ -83,7 +84,8 @@ module Workflow
 
     def self.candidates(workload, rules)
       if rules.empty?
-        candidates = workload.select{|k,v| v.empty? }.
+        candidates = workload.
+          select{|k,v| v.empty? }.
           collect{|k,v| k }.
           reject{|k| k.done? }
       else
@@ -107,7 +109,8 @@ module Workflow
 
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
-    def initialize(timer = 5, available_resources = {})
+    def initialize(timer = 5, available_resources = nil)
+      available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
       @timer               = timer
       @available_resources = IndiferentHash.setup(available_resources)
       @resources_requested = IndiferentHash.setup({})
@@ -155,7 +158,7 @@ module Workflow
         log = job_rules[:log] if job_rules 
         log = Log.severity if log.nil?
         Log.with_severity log do
-          job.produce(false, true)
+          job.produce(false, :nowait)
         end
       end
     end
@@ -228,8 +231,9 @@ module Workflow
 
           new_workload = {}
           workload.each do |k,v|
-            next if k.done?
-            new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
+            next if k.done? || k.error? || k.aborted?
+            #new_workload[k] = v.reject{|d| d.done? || ((d.error? || d.aborted?) && ! d.recoverable_error?)}
+            new_workload[k] = v.reject{|d| d.done? || d.error? || d.aborted?}
           end
           workload = new_workload
           sleep timer

data/lib/rbbt/workflow.rb

@@ -423,7 +423,7 @@ module Workflow
 
     # jobname => true sets the value of the input to the name of the job
     if task.input_options
-      jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
+      jobname_input = task.input_options.select{|i,o| o[:jobname] }.collect{|i,o| i }.first
     else
       jobname_input = nil 
     end
@@ -476,14 +476,14 @@ module Workflow
       end
     end
 
+    input_values = task.take_input_values(inputs)
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
       step_path = step_path taskname, jobname, [], [], extension
-      input_values = task.take_input_values(inputs)
     else
-      input_values = task.take_input_values(inputs)
       step_path = step_path taskname, jobname, input_values, dependencies, extension
     end
 
+
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
@@ -503,8 +503,6 @@ module Workflow
 
   def _job(taskname, jobname = nil, inputs = {})
 
-    _inputs = IndiferentHash.setup(inputs.dup)
-
     task_info = task_info(taskname)
     task_inputs = task_info[:inputs]
     persist_inputs = inputs.values_at(*task_inputs)

data/share/rbbt_commands/workflow/task

@@ -191,7 +191,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
 -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
 --fork Run job asyncronously and monitor progress. It monitors detached processes as well
---orchestrate Run the job through the orchestrator
+--orchestrate* Run the job through the orchestrator
 --detach Run job asyncronously and detach process
 --exec Run job with no persistence
 -O--output* Save job result into file
@@ -441,7 +441,17 @@ begin
     job.fork
   elsif options[:orchestrate]
     require 'rbbt/workflow/util/orchestrator'
-    Workflow::Orchestrator.process job
+    rules = case options[:orchestrate]
+            when 'none', 'open', 'default'
+              nil
+            else
+              YAML.parse(Open.read(options[:orchestrate]))
+            end
+    if rules
+      Workflow::Orchestrator.process rules, job
+    else
+      Workflow::Orchestrator.process job
+    end unless job.done?
   else
     job.run(:stream)
     res = job
@@ -577,10 +587,10 @@ when Step
   elsif detach
     exit! 0
   else
+    res.join
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else
-      res.join
       Open.open(res.path, :mode => 'rb') do |io|
         Misc.consume_stream(io, false, out)
       end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.0
+  version: 5.33.3
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-18 00:00:00.000000000 Z
+date: 2022-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake