rbbt-util 5.32.30 → 5.33.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e173c4373fc6358b63964247a0d81bab2bd632a2e6bce1b93ff9ac1bd6aa53ca
-  data.tar.gz: 26c3352b53615f1e1c057727ac6f129dedbb9fabb09742513b2368b0b2bd7327
+  metadata.gz: 331adb8d6a45a7cc3ee18b17785ffe03d28585274e9b2d1cab5cd502cdc90821
+  data.tar.gz: 8580df0b38d805739fe3ad80e688c92a5c31305893ef1cf6cfccd88c0aaccc7a
 SHA512:
-  metadata.gz: f8d84e234a76a38352786f869aaf5fee4f4b159a402b9241707c18f7cbe595d630a645e75bad7ae512652d664b98c61a13c24beb972e1c55db2b300a8c82744b
-  data.tar.gz: 0fdcfe8522907197eeacfffa0551dbde81db0fc859f0f85eb4281e87df44c9a0b90026423dfad79fd6c9cd8429be655eadc94e94c85cd79d0b971134ea6427ee
+  metadata.gz: 6381f886a7282c03c8e8303ae148fe4a4fc3b82dad60d1d039953d83075aa85ccc19bb75f24811e2158a6715b9ed471288b2b23110c981a61ad167bc6593aac0
+  data.tar.gz: 1476c3fdf9fe56928a9dd85fc2384735b71a0a40a45359197fc203608f2933aa6a6f5ec94c1bd5ae4b5782b2cd2bbe752f2d06973f9989307d81e3a1f261eba2

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -80,6 +80,13 @@ module HPC
       batches.each do |batch|
         jobs = batch[:jobs]
         all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
+
+        minimum = all_deps
+        all_deps.each do |dep|
+          minimum -= job_dependencies(dep)
+        end
+
+        all_deps = minimum
         deps = all_deps.collect do |d|
           (batches - [batch]).select{|batch| batch[:jobs].collect(&:path).include? d.path }
         end.flatten.uniq

data/lib/rbbt/hpc/orchestrate.rb

@@ -14,8 +14,14 @@ module HPC
       options.delete "detach"
       options.delete "jobname"
 
-      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
-      rules ||= {}
+      if options[:orchestration_rules]
+        rules = YAML.load(Open.read(options[:orchestration_rules]))
+      elsif Rbbt.etc.slurm["default.yaml"].exists?
+        rules = YAML.load(Open.read(Rbbt.etc.slurm["default.yaml"]))
+      else
+        rules = {}
+      end
+
       IndiferentHash.setup(rules)
 
       batches = HPC::Orchestration.job_batches(rules, job)
@@ -26,7 +32,29 @@ module HPC
         raise "No batch without unmet dependencies" if top.nil?
         batches.delete top
         job_options = options.merge(top[:rules])
-        job_options.merge!(:batch_dependencies => top[:deps].nil? ? [] : top[:deps].collect{|d| batch_ids[d] })
+
+        if top[:deps].nil?
+          batch_dependencies = [] 
+        else 
+          top_jobs = top[:jobs]
+
+          batch_dependencies = top[:deps].collect{|d| 
+            target = d[:top_level]
+            canfail = false
+
+            top_jobs.each do |job|
+              canfail = true if job.canfail_paths.include?(target.path)
+            end
+
+            if canfail
+              'canfail:' + batch_ids[d].to_s
+            else
+              batch_ids[d].to_s
+            end
+          }
+        end
+
+        job_options.merge!(:batch_dependencies => batch_dependencies )
         job_options.merge!(:manifest => top[:jobs].collect{|d| d.task_signature })
 
         if options[:dry_run]

data/lib/rbbt/persist/tsv/adapter.rb

@@ -111,6 +111,31 @@ module Persist
       end
     end
 
+    def with_read(&block)
+      if read? || write?
+        return yield
+      else
+        read_and_close &block
+      end
+    end
+
+    def with_write(&block)
+      if write?
+        return yield
+      else
+        if self.read?
+          self.write_and_read do
+            return yield
+          end
+        else
+          self.write_and_close do
+            return yield
+          end
+        end
+      end
+    end
+
+
     def read_and_close
       if read? || write?
         begin

data/lib/rbbt/persist.rb

@@ -423,6 +423,8 @@ module Persist
           end
         end
 
+        repo.read
+
         case
         when (keys.length == 1 and keys.first == subkey + 'NIL')
           nil
@@ -430,19 +432,19 @@ module Persist
           []
         when (keys.length == 1 and keys.first =~ /:SINGLE$/)
           key = keys.first
-          values = repo.read_and_close do
+          values = repo.with_read do
             repo[key]
           end
           Annotated.load_tsv_values(key, values, "literal", "annotation_types", "JSON")
         when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
-          repo.read_and_close do
+          repo.with_read do
             keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
               Annotated.load_tsv_values(key, v, "literal", "annotation_types", "JSON")
             }
           end
         when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
-          repo.read_and_close do
+          repo.with_read do
 
             res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
@@ -457,7 +459,7 @@ module Persist
         else
           entities = yield
 
-          repo.write_and_close do 
+          repo.write_and_read do 
             case
             when entities.nil?
               repo[subkey + "NIL"] = nil

data/lib/rbbt/resource/path.rb

@@ -462,4 +462,8 @@ module Path
       Path.setup relative_file , @pkgdir, @resource
     end
   end
+
+  def clean_annotations
+    "" << self.to_s
+  end
 end

data/lib/rbbt/tsv/util.rb

@@ -297,6 +297,8 @@ module TSV
     when :double
       if field.nil?
         through do |k,v| new[k] = v.first end
+      elsif field == :all
+        through do |k,v| new[k] = v.flatten.compact end
       else
         pos = identify_field field
         through do |k,v| new[k] = v[pos] end
@@ -313,7 +315,16 @@ module TSV
       end
     end
     self.annotate(new)
-    new.fields = new.fields[0..0] if new.fields
+    if new.fields
+      case field
+      when nil
+        new.fields = new.fields[0..0]
+      when :all
+        new.fields = [new.fields * "+"]
+      else
+        new.fields = [field]
+      end
+    end
     new.type = :flat
     new
   end

data/lib/rbbt/util/concurrency/processes.rb

@@ -205,7 +205,8 @@ class RbbtProcessQueue
     end
 
     @queue.close_read
-    Log.info "Cpu process (#{num_processes}) started with master: #{@master_pid}"
+    Log.info "Cpu process #{@master_pid} with #{num_processes} workers."
+    Log.low "Signal #{@master_pid} USR1/USR2 (#10/#12) to increase/decrease workers."
   end
 
   def init(&block)

data/lib/rbbt/util/misc/exceptions.rb

@@ -32,8 +32,6 @@ class DependencyError < Aborted
   def initialize(msg)
     if defined? Step and Step === msg
       step = msg
-      workflow = step.path.split("/")[-3]
-      new_msg = [workflow, step.short_path, step.messages.last] * " - "
       new_msg = [step.path, step.messages.last] * ": "
       super(new_msg)
     else
@@ -42,6 +40,21 @@ class DependencyError < Aborted
   end
 end
 
+class DependencyRbbtException < RbbtException
+  def initialize(msg)
+    if defined? Step and Step === msg
+      step = msg
+
+      new_msg = nil
+      new_msg = [step.path, step.messages.last] * ": "
+
+      super(new_msg)
+    else
+      super(msg)
+    end
+  end
+end
+
 class DontClose < Exception; end
 
 class KeepLocked < Exception
@@ -64,4 +77,3 @@ class StopInsist < Exception
     @exception = exception
   end
 end
-

data/lib/rbbt/workflow/step/accessor.rb

@@ -44,7 +44,7 @@ class Step
   end
 
   def self.files_dir(path)
-    path.nil? ? nil : path + '.files'
+    path.nil? ? nil : Path.setup(path + '.files')
   end
 
   def self.info_file(path)
@@ -154,8 +154,12 @@ class Step
     value = Annotated.purge value if defined? Annotated
     Open.lock(info_file, :lock => info_lock) do
       i = info(false).dup
+      value = Annotated.purge(value)
+
       i[key] = value 
+
       dump = Step.serialize_info(i)
+
       @info_cache = IndiferentHash.setup(i)
       Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now

data/lib/rbbt/workflow/step/dependencies.rb

@@ -107,7 +107,26 @@ class Step
       canfail = ComputeDependency === job && job.canfail?
     end
 
-    raise DependencyError, job if job.error? and not canfail
+    raise_dependency_error(job) if job.error? and not canfail
+  end
+
+  def self.raise_dependency_error(job)
+    begin
+      if job.get_exception
+        klass = job.get_exception.class
+      else
+        klass = Kernel.const_get(info[:exception][:class])
+      end
+    rescue
+      Log.exception $!
+      raise DependencyError, job 
+    end
+
+    if (klass <= RbbtException)
+      raise DependencyRbbtException, job 
+    else
+      raise DependencyError, job 
+    end
   end
 
   def log_dependency_exec(dependency, action)
@@ -169,7 +188,7 @@ class Step
 
       if dependency.error?
         log_dependency_exec(dependency, :error)
-        raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
+        raise_dependency_error dependency 
       end
 
       if dependency.streaming?
@@ -368,7 +387,7 @@ class Step
       seen_paths << step.path
       begin
         Step.prepare_for_execution(step) unless step == self
-      rescue DependencyError
+      rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
       next unless step.dependencies and step.dependencies.any?
@@ -493,6 +512,7 @@ class Step
     return true if @overriden
     return true if dependencies.select{|dep| dep.overriden? }.any?
     info[:archived_info].each do |f,i|
+      next if Symbol === i
       return true if i[:overriden] || i["overriden"]
     end if info[:archived_info]
     return false

data/lib/rbbt/workflow/step/info.rb

@@ -101,7 +101,7 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      if dep_info
+      if Hash === dep_info
         dep_info[:inputs].each do |k,v|
           all_inputs[k] = v unless all_inputs.include?(k)
         end if dep_info[:inputs]
@@ -285,7 +285,7 @@ class Step
     begin
       return true unless info[:exception]
       klass = Kernel.const_get(info[:exception][:class])
-      ! (klass <= RbbtException)
+      ! (klass <= RbbtException )
     rescue Exception
       true
     end

data/lib/rbbt/workflow/step/run.rb

@@ -437,7 +437,7 @@ class Step
         end
       end # END SYNC
       res
-    rescue DependencyError
+    rescue DependencyError, DependencyRbbtException
       exception $!
     rescue LockInterrupted
       raise $!
@@ -484,7 +484,7 @@ class Step
     if dofork
       fork(true) unless started?
 
-      join unless done?
+      join unless done? or dofork == :nowait
     else
       run(true) unless started?
 
@@ -496,6 +496,7 @@ class Step
 
   def fork(no_load = false, semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid and Misc.pid_exists?(@pid) and not done? and info[:forked]
+    Log.debug "Fork to run #{self.path}"
     sout, sin = Misc.pipe if no_load == :stream
     @pid = Process.fork do
       Signal.trap(:TERM) do

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -26,7 +26,11 @@ module Workflow
 
         type = :path if file.split(".").last == 'as_path'
 
+        type = :nofile if file.split(".").last == 'nofile'
+
         case type
+        when :nofile
+          inputs[input.to_sym]  = Open.realpath(file)
         when :path
           inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
         when :io

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -65,9 +65,13 @@ module Workflow
 
       IndiferentHash.setup(resources)
 
-      default_resources = rules["default_resources"] || rules["defaults"]["resources"]
+      default_resources = rules["default_resources"] 
+      default_resources ||= rules["defaults"]["resources"] if rules["defaults"]
+      default_resources ||= {}
+
       default_resources.each{|k,v| resources[k] ||= v } if default_resources
 
+      resources = {:cpus => 1} if resources.empty?
       resources
     end
 
@@ -80,7 +84,8 @@ module Workflow
 
     def self.candidates(workload, rules)
       if rules.empty?
-        candidates = workload.select{|k,v| v.empty? }.
+        candidates = workload.
+          select{|k,v| v.empty? }.
           collect{|k,v| k }.
           reject{|k| k.done? }
       else
@@ -98,9 +103,14 @@ module Workflow
       candidates
     end
 
+    def self.process(*args)
+      self.new.process(*args)
+    end
+
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
-    def initialize(timer = 5, available_resources = {})
+    def initialize(timer = 5, available_resources = nil)
+      available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
       @timer               = timer
       @available_resources = IndiferentHash.setup(available_resources)
       @resources_requested = IndiferentHash.setup({})
@@ -148,7 +158,7 @@ module Workflow
         log = job_rules[:log] if job_rules 
         log = Log.severity if log.nil?
         Log.with_severity log do
-          job.produce(false, true)
+          job.produce(false, :nowait)
         end
       end
     end
@@ -176,7 +186,9 @@ module Workflow
       end
     end
 
-    def process(rules, jobs)
+    def process(rules, jobs = nil)
+      jobs, rules = rules, {} if jobs.nil?
+      jobs = [jobs] if Step === jobs
       begin
 
         workload = Orchestrator.workload(jobs)
@@ -219,8 +231,9 @@ module Workflow
 
           new_workload = {}
           workload.each do |k,v|
-            next if k.done?
-            new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
+            next if k.done? || k.error? || k.aborted?
+            #new_workload[k] = v.reject{|d| d.done? || ((d.error? || d.aborted?) && ! d.recoverable_error?)}
+            new_workload[k] = v.reject{|d| d.done? || d.error? || d.aborted?}
           end
           workload = new_workload
           sleep timer

data/lib/rbbt/workflow.rb

@@ -423,7 +423,7 @@ module Workflow
 
     # jobname => true sets the value of the input to the name of the job
     if task.input_options
-      jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
+      jobname_input = task.input_options.select{|i,o| o[:jobname] }.collect{|i,o| i }.first
     else
       jobname_input = nil 
     end
@@ -476,14 +476,14 @@ module Workflow
       end
     end
 
+    input_values = task.take_input_values(inputs)
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
       step_path = step_path taskname, jobname, [], [], extension
-      input_values = task.take_input_values(inputs)
     else
-      input_values = task.take_input_values(inputs)
       step_path = step_path taskname, jobname, input_values, dependencies, extension
     end
 
+
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
@@ -503,8 +503,6 @@ module Workflow
 
   def _job(taskname, jobname = nil, inputs = {})
 
-    _inputs = IndiferentHash.setup(inputs.dup)
-
     task_info = task_info(taskname)
     task_inputs = task_info[:inputs]
     persist_inputs = inputs.values_at(*task_inputs)

data/share/rbbt_commands/workflow/task

@@ -191,6 +191,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
 -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
 --fork Run job asyncronously and monitor progress. It monitors detached processes as well
+--orchestrate* Run the job through the orchestrator
 --detach Run job asyncronously and detach process
 --exec Run job with no persistence
 -O--output* Save job result into file
@@ -438,6 +439,19 @@ begin
     end
 
     job.fork
+  elsif options[:orchestrate]
+    require 'rbbt/workflow/util/orchestrator'
+    rules = case options[:orchestrate]
+            when 'none', 'open', 'default'
+              nil
+            else
+              YAML.parse(Open.read(options[:orchestrate]))
+            end
+    if rules
+      Workflow::Orchestrator.process rules, job
+    else
+      Workflow::Orchestrator.process job
+    end unless job.done?
   else
     job.run(:stream)
     res = job
@@ -573,10 +587,10 @@ when Step
   elsif detach
     exit! 0
   else
+    res.join
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else
-      res.join
       Open.open(res.path, :mode => 'rb') do |io|
         Misc.consume_stream(io, false, out)
       end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.30
+  version: 5.33.2
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-14 00:00:00.000000000 Z
+date: 2022-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake