rbbt-util 5.33.19 → 5.34.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54df9c32ce443c2f2742391738139cc97d38cdfe343058149056ad942e35b4a0
-  data.tar.gz: a603395d628ed6b4ca776c45b377b75bafb4e9d76d009d45636008017efd83b3
+  metadata.gz: d612eb753e090f1221cd4d3b2c17e13e0e87f4cea7bc76c920717ba8055cbe8b
+  data.tar.gz: 9bea5062d3d73982ef5a35526c39ca3208fae9b817233344045144cbd0619920
 SHA512:
-  metadata.gz: 16cfe134071945ab0d8fd16060fc9633c8b7dbbf8189246b3736d24ced7803ff13f33b89071939c16bdff2828d8f588612ecfbb7d5481ea4f46c451be233368b
-  data.tar.gz: 9142954e50c256485563b7147117c35c97e87d2c6aebf4fd28500f28369323b597ba0f7f07393d1c186116744af1200884c0bfa527afdf0c5dee8033b904daa8
+  metadata.gz: 2d1e5c3fe701bb42049ab34bbe74fddb3e9a0a29d292cc8c8fbfc93049b0004be5b449aa83939de8924f2cdf19a720c044ea7491aacc96037e1acd549bd1201f
+  data.tar.gz: 54abc57de1a43b04337d8f9ad27668af2d07db21dbac7ee925728d6058dee280624385ef5b40280c0ec2439b6d80ed350bc136c9bef15116ba56292532d5e210

data/lib/rbbt/hpc/batch.rb

@@ -555,7 +555,7 @@ env > #{batch_options[:fenv]}
 
         batch_job = run_template(batch_dir, dry_run)
 
-        return batch_job unless tail
+        return [batch_job, batch_dir] unless tail
 
         t_monitor = Thread.new do
           self.follow_job(batch_dir, :STDERR)
@@ -570,8 +570,8 @@ env > #{batch_options[:fenv]}
           Open.ln path + '.info', job.path + '.info'  if Open.exists?(path + '.info')
           Open.ln path + '.files', job.path + '.files' if Open.exists?(path + '.files')
         end
-        batch_job
- 
+
+        [batch_job, batch_dir]
       end
     end
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -49,6 +49,8 @@ module HPC
       Log.high "Compute #{batches.length} batches"
 
       batch_ids = {}
+      last_id = nil
+      last_dir = nil
       while batches.any?
         top = batches.select{|b| b[:deps].nil? || (b[:deps] - batch_ids.keys).empty? }.first
         raise "No batch without unmet dependencies" if top.nil?
@@ -87,10 +89,12 @@ module HPC
           puts Log.color(:yellow, "Options: ") + job_options.inspect
           batch_ids[top] = top[:top_level].task_signature
         else
-          id = run_job(top[:top_level], job_options)
-          batch_ids[top] = id
+          id, dir = run_job(top[:top_level], job_options)
+          last_id = batch_ids[top] = id
+          last_dir = dir
         end
       end
+      [last_id, last_dir]
     end
 
   end

data/lib/rbbt/util/misc/inspect.rb

@@ -56,7 +56,8 @@ module Misc
       ":" << obj.to_s
     when String
       if obj.length > 100
-        "'" << obj.slice(0,30) << "<...#{obj.length}...>" << obj.slice(-10,30)<< "'"
+        digest = Misc.digest(obj)
+        "'" << obj.slice(0,30) << "<...#{obj.length} - #{digest[0..4]}...>" << obj.slice(-10,30)<< "'"
       else 
         "'" << obj << "'"
       end

data/lib/rbbt/util/misc/pipes.rb

@@ -92,7 +92,6 @@ module Misc
       ConcurrentStream.setup sout, :pids => [pid]
     else
 
-
       ConcurrentStream.setup sin, :pair => sout
       ConcurrentStream.setup sout, :pair => sin
 
@@ -782,4 +781,13 @@ module Misc
     ConcurrentStream.setup out, :threads => monitor_thread
   end
 
+  def self.open_gz_pipe
+    sout = Misc.open_pipe do |sin|
+      yield sin
+      sin.close
+    end
+
+    Open.gzip(sout)
+  end
+
 end

data/lib/rbbt/util/open.rb

@@ -748,6 +748,7 @@ module Open
           raise $!
         end
         content.close
+        content.join if content.respond_to? :join
       end
     end
 

data/lib/rbbt/workflow/util/provenance.rb

@@ -100,11 +100,13 @@ class Step
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
-          input_name = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+          input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
             d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
-          }.last.first
-          input_dependencies[id] ||= []
-          input_dependencies[id] << [dep, input_name]
+          }.last
+          if input_name
+            input_dependencies[id] ||= []
+            input_dependencies[id] << [dep, input_name]
+          end
         end
       end
     end

data/lib/rbbt/workflow.rb

@@ -414,7 +414,7 @@ module Workflow
     end if task.required_inputs
 
     if missing_inputs.length == 1
-      raise ParameterException, "Input #{missing_inputs.first} is required but was not provided or is nil"
+      raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
     end
 
     if missing_inputs.length > 1

data/share/rbbt_commands/hpc/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
@@ -352,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/hpc/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/hpc/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

data/share/rbbt_commands/lsf/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
@@ -352,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/lsf/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/lsf/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

data/share/rbbt_commands/slurm/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
@@ -352,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/slurm/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/slurm/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

data/share/rbbt_commands/workflow/task

@@ -469,7 +469,7 @@ begin
   end
 
   if options.delete(:printpath)
-    job.join
+    job.join if job.running?
     raise job.messages.last if (job.error? || job.aborted?) && job.messages
     if Open.remote? job.path
       puts job.url + Log.color(:blue, "?_format=raw")

data/test/rbbt/util/misc/test_pipes.rb

@@ -284,4 +284,41 @@ line4
       end
     end
   end
+
+  def test_gz_pipe
+    text =<<-EOF
+line1
+line2
+line3
+line4
+    EOF
+
+    TmpFile.with_file nil, :extension => 'txt.gz' do |file|
+      sout = Misc.open_gz_pipe do |sin|
+        text.split("\n").each do |line|
+          sin.puts line
+        end
+      end
+
+      Open.mkdir File.basename(file)
+      thr1 = Misc.consume_stream(sout, true, file)
+      thr1.join
+      assert Open.gzip?(file)
+      assert_equal text, Open.read(file)
+    end
+  end
+
+  def test_open_pipe_error
+    sout = Misc.open_pipe do |sin|
+      10.times do |i|
+        sin.puts "line #{i}"
+      end
+      raise
+    end
+
+    TmpFile.with_file do |tmp|
+      #Misc.consume_stream(sout, false, tmp)
+      Open.write(tmp, sout)
+    end
+  end
 end

data/test/rbbt/util/test_open.rb

@@ -186,6 +186,6 @@ class TestOpen < Test::Unit::TestCase
     end
 
   end
-  
+
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.19
+  version: 5.34.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-26 00:00:00.000000000 Z
+date: 2022-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake