rbbt-util 5.33.19 → 5.33.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54df9c32ce443c2f2742391738139cc97d38cdfe343058149056ad942e35b4a0
-  data.tar.gz: a603395d628ed6b4ca776c45b377b75bafb4e9d76d009d45636008017efd83b3
+  metadata.gz: 87f3c2c6750c49c90cce3fd4f8e24d270d471538f65374e388ec6ac83bf7134c
+  data.tar.gz: fb4909c5ef2ab2a751271f1f49db270ea1f91c4a84ed26504ddebebf23aae75a
 SHA512:
-  metadata.gz: 16cfe134071945ab0d8fd16060fc9633c8b7dbbf8189246b3736d24ced7803ff13f33b89071939c16bdff2828d8f588612ecfbb7d5481ea4f46c451be233368b
-  data.tar.gz: 9142954e50c256485563b7147117c35c97e87d2c6aebf4fd28500f28369323b597ba0f7f07393d1c186116744af1200884c0bfa527afdf0c5dee8033b904daa8
+  metadata.gz: 39ca36640608cc39e279e0402f55749d5851b3713ee1fabe1eac414ae8ec5bf8f3a7741a71551142aede5ba27c9143b54972720fe321c563ac47b5db4a868da6
+  data.tar.gz: a7f50b0bb6829c781c9b8081b63464997f1d2aa1e8ab68871af6d13799867f4f136385b704e1bf5badce2ed03171db421ba5f9452d491b02c96aa8fe36d8eeea

data/lib/rbbt/hpc/batch.rb

@@ -555,7 +555,7 @@ env > #{batch_options[:fenv]}
 
         batch_job = run_template(batch_dir, dry_run)
 
-        return batch_job unless tail
+        return [batch_job, batch_dir] unless tail
 
         t_monitor = Thread.new do
           self.follow_job(batch_dir, :STDERR)
@@ -570,8 +570,8 @@ env > #{batch_options[:fenv]}
           Open.ln path + '.info', job.path + '.info'  if Open.exists?(path + '.info')
           Open.ln path + '.files', job.path + '.files' if Open.exists?(path + '.files')
         end
-        batch_job
- 
+
+        [batch_job, batch_dir]
       end
     end
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -49,6 +49,8 @@ module HPC
       Log.high "Compute #{batches.length} batches"
 
       batch_ids = {}
+      last_id = nil
+      last_dir = nil
       while batches.any?
         top = batches.select{|b| b[:deps].nil? || (b[:deps] - batch_ids.keys).empty? }.first
         raise "No batch without unmet dependencies" if top.nil?
@@ -87,10 +89,12 @@ module HPC
           puts Log.color(:yellow, "Options: ") + job_options.inspect
           batch_ids[top] = top[:top_level].task_signature
         else
-          id = run_job(top[:top_level], job_options)
-          batch_ids[top] = id
+          id, dir = run_job(top[:top_level], job_options)
+          last_id = batch_ids[top] = id
+          last_dir = dir
         end
       end
+      [last_id, last_dir]
     end
 
   end

data/lib/rbbt/workflow/util/provenance.rb

@@ -100,11 +100,13 @@ class Step
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
-          input_name = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+          input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
             d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
-          }.last.first
-          input_dependencies[id] ||= []
-          input_dependencies[id] << [dep, input_name]
+          }.last
+          if input_name
+            input_dependencies[id] ||= []
+            input_dependencies[id] << [dep, input_name]
+          end
         end
       end
     end

data/share/rbbt_commands/hpc/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 

data/share/rbbt_commands/hpc/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/hpc/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

data/share/rbbt_commands/lsf/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 

data/share/rbbt_commands/lsf/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/lsf/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

data/share/rbbt_commands/slurm/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 

data/share/rbbt_commands/slurm/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/slurm/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.19
+  version: 5.33.20
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-26 00:00:00.000000000 Z
+date: 2022-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake