rbbt-util 5.33.20 → 5.34.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87f3c2c6750c49c90cce3fd4f8e24d270d471538f65374e388ec6ac83bf7134c
-  data.tar.gz: fb4909c5ef2ab2a751271f1f49db270ea1f91c4a84ed26504ddebebf23aae75a
+  metadata.gz: dce81f37830228a43c702909b856ebf661002d17b49614d67b6c04e16d339d4b
+  data.tar.gz: d7cb5ef82e50da2287c4bf9cf3912d66e36c27bb72a1c1c702d86cf7d1e8df78
 SHA512:
-  metadata.gz: 39ca36640608cc39e279e0402f55749d5851b3713ee1fabe1eac414ae8ec5bf8f3a7741a71551142aede5ba27c9143b54972720fe321c563ac47b5db4a868da6
-  data.tar.gz: a7f50b0bb6829c781c9b8081b63464997f1d2aa1e8ab68871af6d13799867f4f136385b704e1bf5badce2ed03171db421ba5f9452d491b02c96aa8fe36d8eeea
+  metadata.gz: fd9ad6b40180ef8cdc5a8e8335f291e557543f14cdd0cbc6760e7dd6f1327a5a73b5be92b1c037ccd46bca8596c86e7bf6eb7221989e9edac52ad34e81ad4e45
+  data.tar.gz: 72798bb53863c93aa768edc6c523d34e682dc374ebf4fe0307590faddb9022237a49f0556de364e7139345b35504b6d19b9c134abf8d0ae751a38c1d2c0331f7

data/lib/rbbt/hpc/orchestrate.rb

@@ -31,6 +31,7 @@ module HPC
       options.delete "detach"
       options.delete "jobname"
       options.delete "load_inputs"
+      options.delete "provenance"
 
       Log.high "Prepare for exec"
       prepare_for_execution(job)

data/lib/rbbt/util/misc/inspect.rb

@@ -56,7 +56,8 @@ module Misc
       ":" << obj.to_s
     when String
       if obj.length > 100
-        "'" << obj.slice(0,30) << "<...#{obj.length}...>" << obj.slice(-10,30)<< "'"
+        digest = Misc.digest(obj)
+        "'" << obj.slice(0,30) << "<...#{obj.length} - #{digest[0..4]}...>" << obj.slice(-10,30)<< "'"
       else 
         "'" << obj << "'"
       end

data/lib/rbbt/util/misc/pipes.rb

@@ -92,7 +92,6 @@ module Misc
       ConcurrentStream.setup sout, :pids => [pid]
     else
 
-
       ConcurrentStream.setup sin, :pair => sout
       ConcurrentStream.setup sout, :pair => sin
 
@@ -782,4 +781,13 @@ module Misc
     ConcurrentStream.setup out, :threads => monitor_thread
   end
 
+  def self.open_gz_pipe
+    sout = Misc.open_pipe do |sin|
+      yield sin
+      sin.close
+    end
+
+    Open.gzip(sout)
+  end
+
 end

data/lib/rbbt/util/open.rb

@@ -678,7 +678,7 @@ module Open
       if Open.exists? notification_file
         key = Open.read(notification_file).strip
         key = nil if key.empty?
-        if key.include?("@")
+        if key && key.include?("@")
           to = from = key
           subject = "Wrote " << file
           message = "Content attached"
@@ -748,6 +748,7 @@ module Open
           raise $!
         end
         content.close
+        content.join if content.respond_to? :join
       end
     end
 

data/lib/rbbt/workflow.rb

@@ -414,7 +414,7 @@ module Workflow
     end if task.required_inputs
 
     if missing_inputs.length == 1
-      raise ParameterException, "Input #{missing_inputs.first} is required but was not provided or is nil"
+      raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
     end
 
     if missing_inputs.length > 1
@@ -542,8 +542,17 @@ module Workflow
     step_path = step_path.call if Proc === step_path
     persist = input_values.nil? ? false : true
     persist = false
+
+    if ! (Path === step_path ? step_path.find : File.exists?(step_path)) && step_path.split("/").length == 3 && File.exists?(new_path = Rbbt.var.jobs[step_path].find)
+      step_path = new_path
+    end
+    
     key = Path === step_path ? step_path.find : step_path
 
+    if ! File.exists?(step_path) && step_path.split("/").length == 3 && File.exists?(new_path = Rbbt.var.jobs[step_path].find)
+      step_path = new_path
+    end
+
     step = Step.new step_path, task, input_values, dependencies
 
     set_step_dependencies(step) unless dependencies
@@ -703,7 +712,15 @@ module Workflow
 
   def self.load_step(path)
     path = Path.setup(path.dup) unless Path === path
-    path = path.find
+
+    if ! (Path === path ? path.exists? : File.exists?(path)) && path.split("/").length == 3 
+      new_path = Rbbt.var.jobs[path]
+      if new_path.exists? || new_path.set_extension('info').exists?
+        path = new_path
+      end
+    end
+
+    path = path.find if Path === path
 
     begin
       _load_step(path)

data/share/rbbt_commands/hpc/list

@@ -356,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/lsf/list

@@ -356,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/slurm/list

@@ -356,5 +356,5 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 end
 
 puts 
-puts "Found #{count} jobs"
+puts Log.color :clear, "Found #{count} jobs"
 

data/share/rbbt_commands/workflow/prov

@@ -36,7 +36,7 @@ file = ARGV.shift
 
 $seen = []
 def get_step(file)
-  file = File.expand_path(file)
+  file = File.expand_path(file) if File.exists?(file)
   file = file.sub(/\.(info|files)$/,'')
   $seen << file
   Workflow.load_step file

data/share/rbbt_commands/workflow/task

@@ -469,7 +469,7 @@ begin
   end
 
   if options.delete(:printpath)
-    job.join
+    job.join if job.running?
     raise job.messages.last if (job.error? || job.aborted?) && job.messages
     if Open.remote? job.path
       puts job.url + Log.color(:blue, "?_format=raw")

data/test/rbbt/util/misc/test_pipes.rb

@@ -284,4 +284,41 @@ line4
       end
     end
   end
+
+  def test_gz_pipe
+    text =<<-EOF
+line1
+line2
+line3
+line4
+    EOF
+
+    TmpFile.with_file nil, :extension => 'txt.gz' do |file|
+      sout = Misc.open_gz_pipe do |sin|
+        text.split("\n").each do |line|
+          sin.puts line
+        end
+      end
+
+      Open.mkdir File.basename(file)
+      thr1 = Misc.consume_stream(sout, true, file)
+      thr1.join
+      assert Open.gzip?(file)
+      assert_equal text, Open.read(file)
+    end
+  end
+
+  def test_open_pipe_error
+    sout = Misc.open_pipe do |sin|
+      10.times do |i|
+        sin.puts "line #{i}"
+      end
+      raise
+    end
+
+    TmpFile.with_file do |tmp|
+      #Misc.consume_stream(sout, false, tmp)
+      Open.write(tmp, sout)
+    end
+  end
 end

data/test/rbbt/util/test_open.rb

@@ -186,6 +186,6 @@ class TestOpen < Test::Unit::TestCase
     end
 
   end
-  
+
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.20
+  version: 5.34.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-30 00:00:00.000000000 Z
+date: 2022-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake