rbbt-util 5.29.0 → 5.29.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 169a7a3ce3ebf35313ca6f64d91e98bdb2bb7df79b0367f6888bf0978c67fb01
-  data.tar.gz: 654b5db6fea8f9ab2df8e8b00aaf945bdc9904965df0489ed72feba136ac6fd6
+  metadata.gz: 9b88fc549c1c1dc5cd56f06d933776d56540e3d0f4bacf77f04a449abcda974f
+  data.tar.gz: caf80ab624418c6c0a744038d98569c3a32d18dc8d9de162b42363e1c281e8c2
 SHA512:
-  metadata.gz: dd168909a2df9e5931f74ca9b91ee64e5ec6b003561d75d2843736321deb1e62ac5807133822be461d370b80ecc8b64885c9e20486548d949d20c90fe72cbbab
-  data.tar.gz: 377c2c070eb32856de9fb09107cc3ab72de64936590db2bb58ca2c4eaf057e954c258f1345d4b542bb39b2aba756f0c37bfdae999b9aacf01267a1bf327d13fa
+  metadata.gz: 8944f1d996afa5610f70046f4e61cf326461050f06f347529cd7e403440e0d9b47d0c33b862bc83e89d4479ce889e773aef0bbabfaf0dae797c3a2164f0e8a8c
+  data.tar.gz: fa0051835f35e23873bde6a075d81805d2878643e35e96e20c5d7423f0c7c48eaf25b2c5afad48215481b0ddd17b9d251555eae5b5c4a7fa420d9796dba9130b

data/lib/rbbt/hpc/orchestrate.rb

@@ -3,6 +3,7 @@ module HPC
   module SLURM
     def self.orchestrate_job(job, options, seen = {})
       return if job.done?
+      return unless job.path.split("/")[-4] == "jobs"
       options.delete "recursive_clean"
       options.delete "tail"
       rules = YAML.load(Open.read(options[:rules])) if options[:rules]

data/lib/rbbt/hpc/slurm.rb

@@ -494,6 +494,7 @@ EOF
       dry_run          = options.delete :dry_run
       tail             = options.delete :tail
       dependencies     = options.delete :slurm_dependencies
+      options[:jobname] = job.clean_name
 
       workflow = job.workflow
 
@@ -518,7 +519,6 @@ EOF
         inputs_dir = File.join(tmp_directory, 'inputs_dir')
         saved = Step.save_job_inputs(job, inputs_dir)
 
-
         if saved && saved.any?
           options[:inputs_dir] = inputs_dir
           cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]

data/lib/rbbt/util/misc/inspect.rb

@@ -287,7 +287,9 @@ module Misc
           when Symbol 
             obj.to_s
           when (defined?(Path) and Path)
-            if Step === obj.resource
+            if defined?(Step) && Open.exists?(Step.info_file(obj))
+              obj2str(Workflow.load_step(obj))
+            elsif defined?(Step) && Step === obj.resource
               "Step file: " + obj
             else
               if obj.exists?
@@ -322,7 +324,11 @@ module Misc
             remove_long_items(obj)
           when File 
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
               "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
@@ -330,7 +336,11 @@ module Misc
             "<IO:" << obj.short_path << ">"
           when IO
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
 
               if obj.respond_to? :obj2str

data/lib/rbbt/workflow/accessor.rb

@@ -16,6 +16,10 @@ end
 
 module Workflow
 
+  def self.job_path?(path)
+    path.split("/")[-4] == "jobs"
+  end
+
   def log(status, message = nil, &block)
     Step.log(status, message, nil, &block)
   end
@@ -301,8 +305,9 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep.workflow = workflow
     dep.info[:name] = dep.name
-    dep.original_task_name ||= dep.task_name
+    dep.original_task_name ||= dep.task_name if dep.workflow
     begin
       workflow = Kernel.const_get workflow if String === workflow
       dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)

data/lib/rbbt/workflow/step/accessor.rb

@@ -94,17 +94,21 @@ class Step
       Log.debug "Saving job input #{name} (#{type}) into #{path}"
       case
       when Step === value
-        Open.link(value.path, path)
+        Open.ln_s(value.path, path)
       when type.to_s == "file"
         if String === value && File.exists?(value)
-          Open.link(value, path)
+          Open.ln_s(value, path)
         else
           Open.write(path + '.yaml', value.to_yaml)
         end
       when Array === value
-        Open.write(path, value * "\n")
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
       when IO === value
-        Open.write(path, value)
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
+        else
+          Open.write(path, value)
+        end
       else
         Open.write(path, value.to_s)
       end
@@ -117,13 +121,17 @@ class Step
     task_name = Symbol === job.overriden ? job.overriden : job.task_name
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
-    task_info = workflow.task_info(task_name)
-    input_types = task_info[:input_types]
-    task_inputs = task_info[:inputs]
-    input_defaults = task_info[:input_defaults]
+    if workflow
+      task_info = workflow.task_info(task_name)
+      input_types = task_info[:input_types]
+      task_inputs = task_info[:inputs]
+      input_defaults = task_info[:input_defaults]
+    else
+      task_info = input_types = task_inputs = input_defaults = {}
+    end
 
     inputs = {}
-    real_inputs = job.real_inputs || job.info[:real_inputs]
+    real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
       next unless real_inputs.include? name.to_sym

data/lib/rbbt/workflow/step/dependencies.rb

@@ -103,7 +103,7 @@ class Step
       end
 
       job.dup_inputs unless status == 'done' or job.started?
-      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
+      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started? || ! Workflow.job_path?(job.path)
 
       canfail = ComputeDependency === job && job.canfail?
     end
@@ -130,7 +130,6 @@ class Step
     (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
   end
 
-
   def execute_dependency(dependency, log = true)
     task_name = self.task_name
     canfail_paths = self.canfail_paths

data/lib/rbbt/workflow/step/run.rb

@@ -122,7 +122,6 @@ class Step
       reject{|dependency| (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dependency) || Open.remote?(dependency.path) }.
       reject{|dependency| dependency.error? }.
       #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
-      #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
       select{|dependency| dependency.updatable? }.
       collect{|dependency| Workflow.relocate_dependency(self, dependency)}
   end

data/lib/rbbt/workflow/util/provenance.rb

@@ -22,11 +22,14 @@ class Step
   end
 
   def self.prov_report_msg(status, name, path, info = nil)
-    parts = path.sub(/\{.*/,'').sub(/#{Regexp.quote(name)}$/,'').split "/"
+    parts = path.sub(/\{.*/,'').split "/"
 
+    parts.pop
+    
     task = Log.color(:yellow, parts.pop)
     workflow = Log.color(:magenta, parts.pop)
-    if status.to_s == 'noinfo' and parts.last != 'jobs'
+    #if status.to_s == 'noinfo' && parts.last != 'jobs'
+    if ! Workflow.job_path?(path)
       task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
     end
 

data/test/rbbt/test_workflow.rb

@@ -386,9 +386,9 @@ class TestWorkflow < Test::Unit::TestCase
     job.run
     Misc.with_env "RBBT_UPDATE", 'true' do
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
-      job = TestWF.job(:t3)
-      job.step(:t1).clean
-      assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      #job = TestWF.job(:t3)
+      #job.step(:t1).clean
+      #assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
       job = TestWF.job(:t3).recursive_clean
       job.run
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
@@ -427,7 +427,7 @@ class TestWorkflow < Test::Unit::TestCase
     TmpFile.with_file do |dir|
       Path.setup(dir)
       Step.save_job_inputs(job, dir)
-      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.read']
+      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.yaml']
       inputs  = Workflow.load_inputs(dir, [:file], :file => :file)
       assert_equal inputs, {:file => 'code'}
     end
@@ -455,9 +455,9 @@ class TestWorkflow < Test::Unit::TestCase
     job.run
     Misc.with_env "RBBT_UPDATE", 'true' do
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
-      job = TestWF.job(:t3)
-      job.step(:t1).clean
-      assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      #job = TestWF.job(:t3)
+      #job.step(:t1).clean
+      #assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
       job = TestWF.job(:t3).recursive_clean
       job.run
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.29.0
+  version: 5.29.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-12 00:00:00.000000000 Z
+date: 2021-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake