rbbt-util 5.33.0 → 5.33.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a6b198e57fe73c0e36e7aea6164bafd1a8a97d10b3a992f17d0ba6dc531a3c7
-  data.tar.gz: 5aa8b8753fc88d5de6fa8f6db31b9b71a57ee5dcc86380041582a4985984f749
+  metadata.gz: 70ba4440be0bdca6f80b4aa78393ca4fbec5dbc8d8a1cc72b6393e73127b5340
+  data.tar.gz: 42021bd242233b5e4e3a268c27291953d84c8bad85b87d8feaf6389685f043bf
 SHA512:
-  metadata.gz: a61b6e6671803d73b7993b2de9c6cf16d4f463ddd04f9378b54c77a00eb7292b4e817eea14cce81704fe3f2640d38ede202a3928d293ab1fe7e0e67fa87a35b0
-  data.tar.gz: d41b1477aab34bfc08610602a3e333a5701edb58d9fa10ecf24436659c4e996bb000ceccb20bbcba1f153ec4e888264ae47ab28689b9e2879601da0b7822e831
+  metadata.gz: d6a229aa39440cf3b0b42a505e32c699d791d8dd1e1328f42cd6500c947560b0d9343312080af78475a6f4835077e53c029d862e9d4c90addd84408925754b50
+  data.tar.gz: e31505e92ca6425095a90bdcd8fe95d435c8435bc929dcb1c0c77083a1286439c952d7a98af1091664fcee7a57ba3f86b99258a84958714d630ab4a3678f1ee3

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -80,6 +80,13 @@ module HPC
       batches.each do |batch|
         jobs = batch[:jobs]
         all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
+
+        minimum = all_deps
+        all_deps.each do |dep|
+          minimum -= job_dependencies(dep)
+        end
+
+        all_deps = minimum
         deps = all_deps.collect do |d|
           (batches - [batch]).select{|batch| batch[:jobs].collect(&:path).include? d.path }
         end.flatten.uniq

data/lib/rbbt/hpc/orchestrate.rb

@@ -26,7 +26,29 @@ module HPC
         raise "No batch without unmet dependencies" if top.nil?
         batches.delete top
         job_options = options.merge(top[:rules])
-        job_options.merge!(:batch_dependencies => top[:deps].nil? ? [] : top[:deps].collect{|d| batch_ids[d] })
+
+        if top[:deps].nil?
+          batch_dependencies = [] 
+        else 
+          top_jobs = top[:jobs]
+
+          batch_dependencies = top[:deps].collect{|d| 
+            target = d[:top_level]
+            canfail = false
+
+            top_jobs.each do |job|
+              canfail = true if job.canfail_paths.include?(target.path)
+            end
+
+            if canfail
+              'canfail:' + batch_ids[d].to_s
+            else
+              batch_ids[d].to_s
+            end
+          }
+        end
+
+        job_options.merge!(:batch_dependencies => batch_dependencies )
         job_options.merge!(:manifest => top[:jobs].collect{|d| d.task_signature })
 
         if options[:dry_run]

data/lib/rbbt/util/misc/exceptions.rb

@@ -32,8 +32,6 @@ class DependencyError < Aborted
   def initialize(msg)
     if defined? Step and Step === msg
       step = msg
-      workflow = step.path.split("/")[-3]
-      new_msg = [workflow, step.short_path, step.messages.last] * " - "
       new_msg = [step.path, step.messages.last] * ": "
       super(new_msg)
     else
@@ -42,6 +40,21 @@ class DependencyError < Aborted
   end
 end
 
+class DependencyRbbtException < RbbtException
+  def initialize(msg)
+    if defined? Step and Step === msg
+      step = msg
+
+      new_msg = nil
+      new_msg = [step.path, step.messages.last] * ": "
+
+      super(new_msg)
+    else
+      super(msg)
+    end
+  end
+end
+
 class DontClose < Exception; end
 
 class KeepLocked < Exception
@@ -64,4 +77,3 @@ class StopInsist < Exception
     @exception = exception
   end
 end
-

data/lib/rbbt/workflow/step/dependencies.rb

@@ -107,7 +107,26 @@ class Step
       canfail = ComputeDependency === job && job.canfail?
     end
 
-    raise DependencyError, job if job.error? and not canfail
+    raise_dependency_error(job) if job.error? and not canfail
+  end
+
+  def self.raise_dependency_error(job)
+    begin
+      if job.get_exception
+        klass = job.get_exception.class
+      else
+        klass = Kernel.const_get(info[:exception][:class])
+      end
+    rescue
+      Log.exception $!
+      raise DependencyError, job 
+    end
+
+    if (klass <= RbbtException)
+      raise DependencyRbbtException, job 
+    else
+      raise DependencyError, job 
+    end
   end
 
   def log_dependency_exec(dependency, action)
@@ -169,7 +188,7 @@ class Step
 
       if dependency.error?
         log_dependency_exec(dependency, :error)
-        raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
+        raise_dependency_error dependency 
       end
 
       if dependency.streaming?
@@ -368,7 +387,7 @@ class Step
       seen_paths << step.path
       begin
         Step.prepare_for_execution(step) unless step == self
-      rescue DependencyError
+      rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
       next unless step.dependencies and step.dependencies.any?

data/lib/rbbt/workflow/step/info.rb

@@ -285,7 +285,7 @@ class Step
     begin
       return true unless info[:exception]
       klass = Kernel.const_get(info[:exception][:class])
-      ! (klass <= RbbtException)
+      ! (klass <= RbbtException )
     rescue Exception
       true
     end

data/lib/rbbt/workflow/step/run.rb

@@ -437,7 +437,7 @@ class Step
         end
       end # END SYNC
       res
-    rescue DependencyError
+    rescue DependencyError, DependencyRbbtException
       exception $!
     rescue LockInterrupted
       raise $!
@@ -484,7 +484,7 @@ class Step
     if dofork
       fork(true) unless started?
 
-      join unless done?
+      join unless done? or dofork == :nowait
     else
       run(true) unless started?
 
@@ -496,6 +496,7 @@ class Step
 
   def fork(no_load = false, semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid and Misc.pid_exists?(@pid) and not done? and info[:forked]
+    Log.debug "Fork to run #{self.path}"
     sout, sin = Misc.pipe if no_load == :stream
     @pid = Process.fork do
       Signal.trap(:TERM) do

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -71,6 +71,7 @@ module Workflow
 
       default_resources.each{|k,v| resources[k] ||= v } if default_resources
 
+      resources = {:cpus => 1} if resources.empty?
       resources
     end
 
@@ -107,7 +108,8 @@ module Workflow
 
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
-    def initialize(timer = 5, available_resources = {})
+    def initialize(timer = 5, available_resources = nil)
+      available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
       @timer               = timer
       @available_resources = IndiferentHash.setup(available_resources)
       @resources_requested = IndiferentHash.setup({})
@@ -155,7 +157,7 @@ module Workflow
         log = job_rules[:log] if job_rules 
         log = Log.severity if log.nil?
         Log.with_severity log do
-          job.produce(false, true)
+          job.produce(false, :nowait)
         end
       end
     end
@@ -202,6 +204,7 @@ module Workflow
             when (job.error? || job.aborted?)
               begin
                 if job.recoverable_error?
+                  iif [:CLEAN, job, job.status, job.info[:exception]]
                   job.clean
                   raise TryAgain
                 else
@@ -228,8 +231,9 @@ module Workflow
 
           new_workload = {}
           workload.each do |k,v|
-            next if k.done?
-            new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
+            next if k.done? || k.error? || k.aborted?
+            #new_workload[k] = v.reject{|d| d.done? || ((d.error? || d.aborted?) && ! d.recoverable_error?)}
+            new_workload[k] = v.reject{|d| d.done? || d.error? || d.aborted?}
           end
           workload = new_workload
           sleep timer

data/lib/rbbt/workflow.rb

@@ -423,7 +423,7 @@ module Workflow
 
     # jobname => true sets the value of the input to the name of the job
     if task.input_options
-      jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
+      jobname_input = task.input_options.select{|i,o| o[:jobname] }.collect{|i,o| i }.first
     else
       jobname_input = nil 
     end
@@ -476,14 +476,14 @@ module Workflow
       end
     end
 
+    input_values = task.take_input_values(inputs)
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
       step_path = step_path taskname, jobname, [], [], extension
-      input_values = task.take_input_values(inputs)
     else
-      input_values = task.take_input_values(inputs)
       step_path = step_path taskname, jobname, input_values, dependencies, extension
     end
 
+
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
@@ -503,8 +503,6 @@ module Workflow
 
   def _job(taskname, jobname = nil, inputs = {})
 
-    _inputs = IndiferentHash.setup(inputs.dup)
-
     task_info = task_info(taskname)
     task_inputs = task_info[:inputs]
     persist_inputs = inputs.values_at(*task_inputs)

data/share/rbbt_commands/workflow/task

@@ -191,7 +191,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
 -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
 --fork Run job asyncronously and monitor progress. It monitors detached processes as well
---orchestrate Run the job through the orchestrator
+--orchestrate* Run the job through the orchestrator
 --detach Run job asyncronously and detach process
 --exec Run job with no persistence
 -O--output* Save job result into file
@@ -441,7 +441,17 @@ begin
     job.fork
   elsif options[:orchestrate]
     require 'rbbt/workflow/util/orchestrator'
-    Workflow::Orchestrator.process job
+    rules = case options[:orchestrate]
+            when 'none', 'open', 'default'
+              nil
+            else
+              YAML.parse(Open.read(options[:orchestrate]))
+            end
+    if rules
+      Workflow::Orchestrator.process rules, job
+    else
+      Workflow::Orchestrator.process job
+    end unless job.done?
   else
     job.run(:stream)
     res = job
@@ -577,10 +587,10 @@ when Step
   elsif detach
     exit! 0
   else
+    res.join
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else
-      res.join
       Open.open(res.path, :mode => 'rb') do |io|
         Misc.consume_stream(io, false, out)
       end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.0
+  version: 5.33.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-18 00:00:00.000000000 Z
+date: 2022-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake