rbbt-util 5.32.28 → 5.33.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a680ce3f6d91a1b0ff8060ee18b8e68efbd43df56f238869b30f6f88617d23d9
-  data.tar.gz: d6c8c724f15d4ce4c910ff9db6becbd0587c8dc04598abcc934884a8368dd93e
+  metadata.gz: 70ba4440be0bdca6f80b4aa78393ca4fbec5dbc8d8a1cc72b6393e73127b5340
+  data.tar.gz: 42021bd242233b5e4e3a268c27291953d84c8bad85b87d8feaf6389685f043bf
 SHA512:
-  metadata.gz: 9b90c640f51de96d469af4b442d9ba66aa89a592c7776196058da3b20e13b07ef50f67e07018435b12bed13492c3bf9d9abf4de565902094a3d58be4ab628fd5
-  data.tar.gz: 523d7234c59bc1781d47827daebb07870913b8dec9851110d5c2e0c92441903601965f9a7bb4b494b0271d64e09ad0f3b80b254176ee3495744c2bbf1f34f714
+  metadata.gz: d6a229aa39440cf3b0b42a505e32c699d791d8dd1e1328f42cd6500c947560b0d9343312080af78475a6f4835077e53c029d862e9d4c90addd84408925754b50
+  data.tar.gz: e31505e92ca6425095a90bdcd8fe95d435c8435bc929dcb1c0c77083a1286439c952d7a98af1091664fcee7a57ba3f86b99258a84958714d630ab4a3678f1ee3

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -18,12 +18,24 @@ module HPC
 
     def self.job_workload(job)
       workload = []
+      path_jobs = {}
+
+      path_jobs[job.path] = job
+
       heap = []
-      heap << job
-      while job = heap.pop
+      heap << job.path
+      while job_path = heap.pop
+        job = path_jobs[job_path]
         next if job.done?
         workload << job
-        heap.concat job_dependencies(job)
+
+        deps =  job_dependencies(job)
+
+        deps.each do  |d|
+          path_jobs[d.path] ||= d
+        end
+
+        heap.concat deps.collect(&:path)
         heap.uniq!
       end
       workload.uniq
@@ -68,8 +80,15 @@ module HPC
       batches.each do |batch|
         jobs = batch[:jobs]
         all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
+
+        minimum = all_deps
+        all_deps.each do |dep|
+          minimum -= job_dependencies(dep)
+        end
+
+        all_deps = minimum
         deps = all_deps.collect do |d|
-          (batches - [batch]).select{|batch| batch[:jobs].include? d }
+          (batches - [batch]).select{|batch| batch[:jobs].collect(&:path).include? d.path }
         end.flatten.uniq
         batch[:deps] = deps
       end
@@ -130,7 +149,7 @@ module HPC
     def self.job_batches(rules, job)
       job_chains = self.job_chains(rules, job)
 
-      workload = job_workload(job)
+      workload = job_workload(job).uniq
 
       batches = chain_batches(rules, job_chains, workload)
       batches = add_batch_deps(batches)

data/lib/rbbt/hpc/orchestrate.rb

@@ -26,7 +26,29 @@ module HPC
         raise "No batch without unmet dependencies" if top.nil?
         batches.delete top
         job_options = options.merge(top[:rules])
-        job_options.merge!(:batch_dependencies => top[:deps].nil? ? [] : top[:deps].collect{|d| batch_ids[d] })
+
+        if top[:deps].nil?
+          batch_dependencies = [] 
+        else 
+          top_jobs = top[:jobs]
+
+          batch_dependencies = top[:deps].collect{|d| 
+            target = d[:top_level]
+            canfail = false
+
+            top_jobs.each do |job|
+              canfail = true if job.canfail_paths.include?(target.path)
+            end
+
+            if canfail
+              'canfail:' + batch_ids[d].to_s
+            else
+              batch_ids[d].to_s
+            end
+          }
+        end
+
+        job_options.merge!(:batch_dependencies => batch_dependencies )
         job_options.merge!(:manifest => top[:jobs].collect{|d| d.task_signature })
 
         if options[:dry_run]

data/lib/rbbt/persist/tsv/adapter.rb

@@ -111,6 +111,31 @@ module Persist
       end
     end
 
+    def with_read(&block)
+      if read? || write?
+        return yield
+      else
+        read_and_close &block
+      end
+    end
+
+    def with_write(&block)
+      if write?
+        return yield
+      else
+        if self.read?
+          self.write_and_read do
+            return yield
+          end
+        else
+          self.write_and_close do
+            return yield
+          end
+        end
+      end
+    end
+
+
     def read_and_close
       if read? || write?
         begin

data/lib/rbbt/persist.rb

@@ -423,6 +423,8 @@ module Persist
           end
         end
 
+        repo.read
+
         case
         when (keys.length == 1 and keys.first == subkey + 'NIL')
           nil
@@ -430,19 +432,19 @@ module Persist
           []
         when (keys.length == 1 and keys.first =~ /:SINGLE$/)
           key = keys.first
-          values = repo.read_and_close do
+          values = repo.with_read do
             repo[key]
           end
           Annotated.load_tsv_values(key, values, "literal", "annotation_types", "JSON")
         when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
-          repo.read_and_close do
+          repo.with_read do
             keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
               Annotated.load_tsv_values(key, v, "literal", "annotation_types", "JSON")
             }
           end
         when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
-          repo.read_and_close do
+          repo.with_read do
 
             res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
@@ -457,7 +459,7 @@ module Persist
         else
           entities = yield
 
-          repo.write_and_close do 
+          repo.write_and_read do 
             case
             when entities.nil?
               repo[subkey + "NIL"] = nil

data/lib/rbbt/resource/path.rb

@@ -462,4 +462,8 @@ module Path
       Path.setup relative_file , @pkgdir, @resource
     end
   end
+
+  def clean_annotations
+    "" << self.to_s
+  end
 end

data/lib/rbbt/tsv/util.rb

@@ -297,6 +297,8 @@ module TSV
     when :double
       if field.nil?
         through do |k,v| new[k] = v.first end
+      elsif field == :all
+        through do |k,v| new[k] = v.flatten.compact end
       else
         pos = identify_field field
         through do |k,v| new[k] = v[pos] end
@@ -313,7 +315,16 @@ module TSV
       end
     end
     self.annotate(new)
-    new.fields = new.fields[0..0] if new.fields
+    if new.fields
+      case field
+      when nil
+        new.fields = new.fields[0..0]
+      when :all
+        new.fields = [new.fields * "+"]
+      else
+        new.fields = [field]
+      end
+    end
     new.type = :flat
     new
   end

data/lib/rbbt/util/concurrency/processes.rb

@@ -205,7 +205,8 @@ class RbbtProcessQueue
     end
 
     @queue.close_read
-    Log.info "Cpu process (#{num_processes}) started with master: #{@master_pid}"
+    Log.info "Cpu process #{@master_pid} with #{num_processes} workers."
+    Log.low "Signal #{@master_pid} USR1/USR2 (#10/#12) to increase/decrease workers."
   end
 
   def init(&block)

data/lib/rbbt/util/migrate.rb

@@ -49,7 +49,7 @@ puts resource[path].find(search_path)
     test_str = options[:test] ? '-nv' : ''
 
     real_paths.each do |source_path|
-      Log.medium "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}"
+      Log.medium "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}" if options[:files]
       if File.directory?(source_path) || source_path =~ /\/$/
         source_path += "/" unless source_path[-1] == "/"
         target += "/" unless target[-1] == "/"

data/lib/rbbt/util/misc/exceptions.rb

@@ -32,8 +32,6 @@ class DependencyError < Aborted
   def initialize(msg)
     if defined? Step and Step === msg
       step = msg
-      workflow = step.path.split("/")[-3]
-      new_msg = [workflow, step.short_path, step.messages.last] * " - "
       new_msg = [step.path, step.messages.last] * ": "
       super(new_msg)
     else
@@ -42,6 +40,21 @@ class DependencyError < Aborted
   end
 end
 
+class DependencyRbbtException < RbbtException
+  def initialize(msg)
+    if defined? Step and Step === msg
+      step = msg
+
+      new_msg = nil
+      new_msg = [step.path, step.messages.last] * ": "
+
+      super(new_msg)
+    else
+      super(msg)
+    end
+  end
+end
+
 class DontClose < Exception; end
 
 class KeepLocked < Exception
@@ -64,4 +77,3 @@ class StopInsist < Exception
     @exception = exception
   end
 end
-

data/lib/rbbt/workflow/step/accessor.rb

@@ -44,7 +44,7 @@ class Step
   end
 
   def self.files_dir(path)
-    path.nil? ? nil : path + '.files'
+    path.nil? ? nil : Path.setup(path + '.files')
   end
 
   def self.info_file(path)
@@ -154,8 +154,12 @@ class Step
     value = Annotated.purge value if defined? Annotated
     Open.lock(info_file, :lock => info_lock) do
       i = info(false).dup
+      value = Annotated.purge(value)
+
       i[key] = value 
+
       dump = Step.serialize_info(i)
+
       @info_cache = IndiferentHash.setup(i)
       Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now

data/lib/rbbt/workflow/step/dependencies.rb

@@ -107,7 +107,26 @@ class Step
       canfail = ComputeDependency === job && job.canfail?
     end
 
-    raise DependencyError, job if job.error? and not canfail
+    raise_dependency_error(job) if job.error? and not canfail
+  end
+
+  def self.raise_dependency_error(job)
+    begin
+      if job.get_exception
+        klass = job.get_exception.class
+      else
+        klass = Kernel.const_get(info[:exception][:class])
+      end
+    rescue
+      Log.exception $!
+      raise DependencyError, job 
+    end
+
+    if (klass <= RbbtException)
+      raise DependencyRbbtException, job 
+    else
+      raise DependencyError, job 
+    end
   end
 
   def log_dependency_exec(dependency, action)
@@ -169,7 +188,7 @@ class Step
 
       if dependency.error?
         log_dependency_exec(dependency, :error)
-        raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
+        raise_dependency_error dependency 
       end
 
       if dependency.streaming?
@@ -368,7 +387,7 @@ class Step
       seen_paths << step.path
       begin
         Step.prepare_for_execution(step) unless step == self
-      rescue DependencyError
+      rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
       next unless step.dependencies and step.dependencies.any?

data/lib/rbbt/workflow/step/info.rb

@@ -101,7 +101,7 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      if dep_info
+      if Hash === dep_info
         dep_info[:inputs].each do |k,v|
           all_inputs[k] = v unless all_inputs.include?(k)
         end if dep_info[:inputs]
@@ -285,7 +285,7 @@ class Step
     begin
       return true unless info[:exception]
       klass = Kernel.const_get(info[:exception][:class])
-      ! (klass <= RbbtException)
+      ! (klass <= RbbtException )
     rescue Exception
       true
     end

data/lib/rbbt/workflow/step/run.rb

@@ -437,7 +437,7 @@ class Step
         end
       end # END SYNC
       res
-    rescue DependencyError
+    rescue DependencyError, DependencyRbbtException
       exception $!
     rescue LockInterrupted
       raise $!
@@ -484,7 +484,7 @@ class Step
     if dofork
       fork(true) unless started?
 
-      join unless done?
+      join unless done? or dofork == :nowait
     else
       run(true) unless started?
 
@@ -496,6 +496,7 @@ class Step
 
   def fork(no_load = false, semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid and Misc.pid_exists?(@pid) and not done? and info[:forked]
+    Log.debug "Fork to run #{self.path}"
     sout, sin = Misc.pipe if no_load == :stream
     @pid = Process.fork do
       Signal.trap(:TERM) do

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -24,7 +24,15 @@ module Workflow
 
         type = :io if file.split(".").last == 'as_io'
 
+        type = :path if file.split(".").last == 'as_path'
+
+        type = :nofile if file.split(".").last == 'nofile'
+
         case type
+        when :nofile
+          inputs[input.to_sym]  = Open.realpath(file)
+        when :path
+          inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
         when :io
           inputs[input.to_sym] = Open.open(Open.realpath(file))
         when :file, :binary
@@ -96,12 +104,21 @@ class Step
         Open.write(path, value.to_s)
       when Step === value
         Open.ln_s(value.path, path)
+      when type.to_s == "binary"
+        if String === value && File.exists?(value)
+          value = File.expand_path(value)
+          Open.ln_s(value, path)
+        elsif String === value && Misc.is_filename?(value, false)
+          Open.write(path + '.as_path' , value)
+        else
+          Open.write(path, value, :mode => 'wb')
+        end
       when type.to_s == "file"
         if String === value && File.exists?(value)
           value = File.expand_path(value)
           Open.ln_s(value, path)
         else
-          value = value.collect{|v| v = "#{v}" if Path === v; v }if Array === value
+          value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
           value = "#{value}" if Path === value
           Open.write(path + '.yaml', value.to_yaml)
         end

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -65,9 +65,13 @@ module Workflow
 
       IndiferentHash.setup(resources)
 
-      default_resources = rules["default_resources"] || rules["defaults"]["resources"]
+      default_resources = rules["default_resources"] 
+      default_resources ||= rules["defaults"]["resources"] if rules["defaults"]
+      default_resources ||= {}
+
       default_resources.each{|k,v| resources[k] ||= v } if default_resources
 
+      resources = {:cpus => 1} if resources.empty?
       resources
     end
 
@@ -98,9 +102,14 @@ module Workflow
       candidates
     end
 
+    def self.process(*args)
+      self.new.process(*args)
+    end
+
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
-    def initialize(timer = 5, available_resources = {})
+    def initialize(timer = 5, available_resources = nil)
+      available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
       @timer               = timer
       @available_resources = IndiferentHash.setup(available_resources)
       @resources_requested = IndiferentHash.setup({})
@@ -148,7 +157,7 @@ module Workflow
         log = job_rules[:log] if job_rules 
         log = Log.severity if log.nil?
         Log.with_severity log do
-          job.produce(false, true)
+          job.produce(false, :nowait)
         end
       end
     end
@@ -176,7 +185,9 @@ module Workflow
       end
     end
 
-    def process(rules, jobs)
+    def process(rules, jobs = nil)
+      jobs, rules = rules, {} if jobs.nil?
+      jobs = [jobs] if Step === jobs
       begin
 
         workload = Orchestrator.workload(jobs)
@@ -193,6 +204,7 @@ module Workflow
             when (job.error? || job.aborted?)
               begin
                 if job.recoverable_error?
+                  iif [:CLEAN, job, job.status, job.info[:exception]]
                   job.clean
                   raise TryAgain
                 else
@@ -219,8 +231,9 @@ module Workflow
 
           new_workload = {}
           workload.each do |k,v|
-            next if k.done?
-            new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
+            next if k.done? || k.error? || k.aborted?
+            #new_workload[k] = v.reject{|d| d.done? || ((d.error? || d.aborted?) && ! d.recoverable_error?)}
+            new_workload[k] = v.reject{|d| d.done? || d.error? || d.aborted?}
           end
           workload = new_workload
           sleep timer

data/lib/rbbt/workflow.rb

@@ -423,7 +423,7 @@ module Workflow
 
     # jobname => true sets the value of the input to the name of the job
     if task.input_options
-      jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
+      jobname_input = task.input_options.select{|i,o| o[:jobname] }.collect{|i,o| i }.first
     else
       jobname_input = nil 
     end
@@ -476,14 +476,14 @@ module Workflow
       end
     end
 
+    input_values = task.take_input_values(inputs)
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
       step_path = step_path taskname, jobname, [], [], extension
-      input_values = task.take_input_values(inputs)
     else
-      input_values = task.take_input_values(inputs)
       step_path = step_path taskname, jobname, input_values, dependencies, extension
     end
 
+
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
@@ -503,8 +503,6 @@ module Workflow
 
   def _job(taskname, jobname = nil, inputs = {})
 
-    _inputs = IndiferentHash.setup(inputs.dup)
-
     task_info = task_info(taskname)
     task_inputs = task_info[:inputs]
     persist_inputs = inputs.values_at(*task_inputs)

data/share/rbbt_commands/workflow/task

@@ -191,6 +191,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
 -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
 --fork Run job asyncronously and monitor progress. It monitors detached processes as well
+--orchestrate* Run the job through the orchestrator
 --detach Run job asyncronously and detach process
 --exec Run job with no persistence
 -O--output* Save job result into file
@@ -438,6 +439,19 @@ begin
     end
 
     job.fork
+  elsif options[:orchestrate]
+    require 'rbbt/workflow/util/orchestrator'
+    rules = case options[:orchestrate]
+            when 'none', 'open', 'default'
+              nil
+            else
+              YAML.parse(Open.read(options[:orchestrate]))
+            end
+    if rules
+      Workflow::Orchestrator.process rules, job
+    else
+      Workflow::Orchestrator.process job
+    end unless job.done?
   else
     job.run(:stream)
     res = job
@@ -573,10 +587,10 @@ when Step
   elsif detach
     exit! 0
   else
+    res.join
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else
-      res.join
       Open.open(res.path, :mode => 'rb') do |io|
         Misc.consume_stream(io, false, out)
       end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.28
+  version: 5.33.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-11 00:00:00.000000000 Z
+date: 2022-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake