rbbt-util 5.33.18 → 5.33.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 972b137af2c124d5288389d49e188d3973246a912c0405c391cdb6b8a5987cae
-  data.tar.gz: 4744dc8f03a8230ad8d5f61f1d15f14c4e5635d5e3798be584083217269b88a5
+  metadata.gz: 563841b942e1b0cf4a97bed6900bd51c96df63ec993d3a5280faadbf7b8eb753
+  data.tar.gz: c2cd78f5dfe5061f97021fd2121dd415d7b6c36c1fef2fc9f496c2cc985e4c4d
 SHA512:
-  metadata.gz: 34f7cbdd329d23341cc6591bfe19f584a4c50862c8fae6c2ccd8d72bcd76fbb129acbaa3bbb03e08eb633aa475d5d0eacb1a015e667db169ebf1f09e588ab579
-  data.tar.gz: 8110363114c606214d115ffb6475ce277dc3f605c1e8eafd97911a53cc259386ed6173d45cca4a5692b8e4ca0a89db71a77b6fb5eca69400b565b820a83a8d56
+  metadata.gz: 6382e73cdb35ff7ab7dcd3de894a192353c39edef06ddaeb030ec376089b638acb0f62f468153433dfb0e3b0242a44146bfaeb1770ef35ee7175200caeea5e12
+  data.tar.gz: '08b8ce8d556f25269c671b6307972669927b468975c77d188b572e2285627968989a2db68de7fa2983908cb7d4cf3a236f0908a78d3917ea46ad84f82c60f536'

data/lib/rbbt/hpc/batch.rb

@@ -555,7 +555,7 @@ env > #{batch_options[:fenv]}
 
         batch_job = run_template(batch_dir, dry_run)
 
-        return batch_job unless tail
+        return [batch_job, batch_dir] unless tail
 
         t_monitor = Thread.new do
           self.follow_job(batch_dir, :STDERR)
@@ -570,8 +570,8 @@ env > #{batch_options[:fenv]}
           Open.ln path + '.info', job.path + '.info'  if Open.exists?(path + '.info')
           Open.ln path + '.files', job.path + '.files' if Open.exists?(path + '.files')
         end
-        batch_job
- 
+
+        [batch_job, batch_dir]
       end
     end
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -49,6 +49,8 @@ module HPC
       Log.high "Compute #{batches.length} batches"
 
       batch_ids = {}
+      last_id = nil
+      last_dir = nil
       while batches.any?
         top = batches.select{|b| b[:deps].nil? || (b[:deps] - batch_ids.keys).empty? }.first
         raise "No batch without unmet dependencies" if top.nil?
@@ -87,10 +89,12 @@ module HPC
           puts Log.color(:yellow, "Options: ") + job_options.inspect
           batch_ids[top] = top[:top_level].task_signature
         else
-          id = run_job(top[:top_level], job_options)
-          batch_ids[top] = id
+          id, dir = run_job(top[:top_level], job_options)
+          last_id = batch_ids[top] = id
+          last_dir = dir
         end
       end
+      [last_id, last_dir]
     end
 
   end

data/lib/rbbt/util/misc/pipes.rb

@@ -92,7 +92,6 @@ module Misc
       ConcurrentStream.setup sout, :pids => [pid]
     else
 
-
       ConcurrentStream.setup sin, :pair => sout
       ConcurrentStream.setup sout, :pair => sin
 
@@ -782,4 +781,13 @@ module Misc
     ConcurrentStream.setup out, :threads => monitor_thread
   end
 
+  def self.open_gz_pipe
+    sout = Misc.open_pipe do |sin|
+      yield sin
+      sin.close
+    end
+
+    Open.gzip(sout)
+  end
+
 end

data/lib/rbbt/util/misc/system.rb

@@ -94,10 +94,10 @@ module Misc
   end
 
   def self.is_filename?(string, need_to_exists = true)
-    return false if string.nil? or string.include? "\n"
+    return false if string.nil?
     return true if defined? Path and Path === string
     return true if string.respond_to? :exists
-    return true if String === string and string.split("/").select{|p| p.length > 265}.empty? and (! need_to_exists || File.exist?(string))
+    return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265}.empty? and (! need_to_exists || File.exist?(string))
     return false
   end
 

data/lib/rbbt/util/open.rb

@@ -748,6 +748,7 @@ module Open
           raise $!
         end
         content.close
+        content.join if content.respond_to? :join
       end
     end
 

data/lib/rbbt/workflow/util/provenance.rb

@@ -100,11 +100,13 @@ class Step
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
-          input_name = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+          input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
             d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
-          }.last.first
-          input_dependencies[id] ||= []
-          input_dependencies[id] << [dep, input_name]
+          }.last
+          if input_name
+            input_dependencies[id] ||= []
+            input_dependencies[id] << [dep, input_name]
+          end
         end
       end
     end

data/lib/rbbt/workflow.rb

@@ -414,7 +414,7 @@ module Workflow
     end if task.required_inputs
 
     if missing_inputs.length == 1
-      raise ParameterException, "Input #{missing_inputs.first} is required but was not provided or is nil"
+      raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
     end
 
     if missing_inputs.length > 1

data/share/rbbt_commands/hpc/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
@@ -352,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/hpc/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/hpc/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

data/share/rbbt_commands/lsf/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
@@ -352,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/lsf/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/lsf/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

data/share/rbbt_commands/slurm/list

@@ -193,7 +193,11 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    if different_system
      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
    else
-     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+       (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:green, id) : Log.color(:yellow, id) ) : 
+        Log.color(:red, id) )
    end 
    prog_rep = []
    if options[:progress] 
@@ -204,7 +208,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
        step = Step.new step_path
        step.load_dependencies_from_info
        has_bar = false
-       (step.rec_dependencies + [step]).reverse.each do |j|
+       [step].reverse.each do |j|
          next if j.done? 
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
@@ -352,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/slurm/orchestrate

@@ -49,17 +49,27 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        if no_load
+          self
         else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
+          join
+          self.load
         end
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ['--detach']

data/share/rbbt_commands/slurm/task

@@ -47,17 +47,18 @@ HPC::BATCH_MODULE = HPC.batch_system batch_system
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+      rescue HPC::SBATCH
+      end
     end
+  end
 end
 
 ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]

data/share/rbbt_commands/workflow/task

@@ -469,7 +469,7 @@ begin
   end
 
   if options.delete(:printpath)
-    job.join
+    job.join if job.running?
     raise job.messages.last if (job.error? || job.aborted?) && job.messages
     if Open.remote? job.path
       puts job.url + Log.color(:blue, "?_format=raw")

data/test/rbbt/util/misc/test_pipes.rb

@@ -284,4 +284,41 @@ line4
       end
     end
   end
+
+  def test_gz_pipe
+    text =<<-EOF
+line1
+line2
+line3
+line4
+    EOF
+
+    TmpFile.with_file nil, :extension => 'txt.gz' do |file|
+      sout = Misc.open_gz_pipe do |sin|
+        text.split("\n").each do |line|
+          sin.puts line
+        end
+      end
+
+      Open.mkdir File.basename(file)
+      thr1 = Misc.consume_stream(sout, true, file)
+      thr1.join
+      assert Open.gzip?(file)
+      assert_equal text, Open.read(file)
+    end
+  end
+
+  def test_open_pipe_error
+    sout = Misc.open_pipe do |sin|
+      10.times do |i|
+        sin.puts "line #{i}"
+      end
+      raise
+    end
+
+    TmpFile.with_file do |tmp|
+      #Misc.consume_stream(sout, false, tmp)
+      Open.write(tmp, sout)
+    end
+  end
 end

data/test/rbbt/util/test_open.rb

@@ -186,6 +186,6 @@ class TestOpen < Test::Unit::TestCase
     end
 
   end
-  
+
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.18
+  version: 5.33.21
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-25 00:00:00.000000000 Z
+date: 2022-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake