rbbt-util 5.32.13 → 5.32.18

data/lib/rbbt/workflow/usage.rb

@@ -67,11 +67,13 @@ end
 
 module Workflow
 
-  def dep_tree(name)
+  def dep_tree(name, seen = [])
     @dep_tree ||= {}
     @dep_tree[name] ||= begin
                         dep_tree = {}
                         self.task_dependencies[name.to_sym].reverse.each do |dep|
+                          next if seen.include? dep
+                          seen << dep
                           dep = dep.first if Array === dep && dep.length == 1
                           dep = dep.dependency if DependencyBlock === dep
 
@@ -87,7 +89,7 @@ module Workflow
 
                           key = [workflow, task]
 
-                          dep_tree[key] = workflow.dep_tree(task)
+                          dep_tree[key] = workflow.dep_tree(task, seen)
                         end if name && self.task_dependencies[name.to_sym]
                         dep_tree
                       end

data/lib/rbbt/workflow/util/data.rb

@@ -23,7 +23,11 @@ module Workflow
         task_info = workflow.task_info(oname)
         dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
         data_options = block.call data_dir, dir_options, task_info
-        {:inputs => data_options.merge(options)}
+        if data_options.include?(:inputs)
+          data_options
+        else
+          {:inputs => data_options.merge(options)}
+        end
       end
     end
   end

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -81,7 +81,7 @@ module Workflow
     def self.candidates(workload, rules)
       if rules.empty?
         candidates = workload.select{|k,v| v.empty? }.
-          collect{|k,v| k}.
+          collect{|k,v| k }.
           reject{|k| k.done? }
       else
         candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.

data/lib/rbbt/workflow/util/provenance.rb

@@ -26,7 +26,7 @@ class Step
     Log.color(color, status.to_s)
   end
 
-  def self.prov_report_msg(status, name, path, info = nil)
+  def self.prov_report_msg(status, name, path, info, input = nil)
     parts = path.sub(/\{.*/,'').split "/"
 
     parts.pop
@@ -43,10 +43,19 @@ class Step
                  rescue Exception
                    nil
                  end
+
+    if input.nil? || input.empty?
+      input_str = nil
+    else
+      input = input.reject{|dep,name| (input & dep.dependencies.collect{|d| [d,name]}).any? }
+      input = input.reject{|dep,name| (input & dep.input_dependencies.collect{|d| [d,name]}).any? }
+      input_str = Log.color(:magenta, "-> ") + input.collect{|dep,name| Log.color(:yellow, dep.task_name.to_s) + ":" + Log.color(:yellow, name) }.uniq * " "
+    end
+
     str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
-            prov_status_msg(status.to_s) << " " << [workflow, task, path].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
           else
-            prov_status_msg(status.to_s) << " " << [workflow, task, path].compact * " " 
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " 
           end
 
     if $inputs and $inputs.any? 
@@ -74,7 +83,7 @@ class Step
     str << "\n"
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false)
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -84,14 +93,33 @@ class Step
     status = :unsync if status == :done and not Open.exist?(path)
     status = :notfound if status == :noinfo and not Open.exist?(path)
 
-    str = " " * offset
-    str << prov_report_msg(status, name, path, info)
-    step.dependencies.reverse.each do |dep|
+
+    this_step_msg = prov_report_msg(status, name, path, info, input)
+
+    input_dependencies = {}
+    step.dependencies.each do |dep|
+      if dep.input_dependencies.any?
+        dep.input_dependencies.each do |id|
+          input_name = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+            d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
+          }.last.first
+          input_dependencies[id] ||= []
+          input_dependencies[id] << [dep, input_name]
+        end
+      end
+    end
+
+    str = ""
+    str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
+    step.dependencies.dup.tap{|l| 
+      l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    }.each do |dep|
       path = dep.path
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen, expand_repeats)
+        str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
       else
         if expand_repeats
           str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
@@ -103,10 +131,13 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
         end
       end
     end if step.dependencies
+
+    str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
     str
   end
 end

data/python/rbbt.py

@@ -1,3 +1,7 @@
+import warnings
+import sys
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
 def rbbt():
     print("Rbbt")
-

data/share/rbbt_commands/hpc/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/hpc/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/lsf/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/lsf/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/slurm/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/slurm/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/system/clean

@@ -34,6 +34,7 @@ force = options.delete :force
 dirty = options.delete :dirty
 time = options.delete :older
 
+ENV["RBBT_DEBUG_CLEAN"] = 'false'
 ENV["RBBT_UPDATE"] = 'true' if dirty
 
 time = Misc.timespan time, 'd' if time

data/share/rbbt_commands/workflow/prov

@@ -131,6 +131,6 @@ if options[:plot]
   end
 
 else
-  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats]).strip
+  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats])
 end
 

data/test/rbbt/workflow/util/test_data.rb

@@ -5,15 +5,24 @@ module TestDataWF
   extend Workflow
   extend Workflow::Data
 
+  input :salutation, :string
+  task :salute_luis => :string do |name,salutation|
+    "Hi Luis: #{salutation}"
+  end
+
   input :name, :string
   input :salutation, :string
   task :salute => :string do |name,salutation|
     "Hi #{name}: #{salutation}"
   end
 
-  data_task :salute_data, TestDataWF, :salute do |directory,options|
+  data_task :salute_data, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
     options.merge({:salutation => directory.salutation.read})
   end
+
+  data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
+    {:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
+  end
 end
 
 class TestWorkflowData < Test::Unit::TestCase
@@ -28,7 +37,11 @@ class TestWorkflowData < Test::Unit::TestCase
 
       job = TestDataWF.job(:salute_data, "TestDir")
       job.recursive_clean.run
-      ppp job.run
+      assert job.run.include? "Miguel"
+
+      job = TestDataWF.job(:salute_data2, "TestDir")
+      job.recursive_clean.run
+      assert job.run.include? "Luis"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.13
+  version: 5.32.18
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-26 00:00:00.000000000 Z
+date: 2021-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -322,6 +322,7 @@ files:
 - lib/rbbt/workflow/accessor.rb
 - lib/rbbt/workflow/annotate.rb
 - lib/rbbt/workflow/definition.rb
+- lib/rbbt/workflow/dependencies.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
 - lib/rbbt/workflow/integration/ansible.rb
@@ -339,8 +340,10 @@ files:
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/accessor.rb
 - lib/rbbt/workflow/step/dependencies.rb
+- lib/rbbt/workflow/step/info.rb
 - lib/rbbt/workflow/step/prepare.rb
 - lib/rbbt/workflow/step/run.rb
+- lib/rbbt/workflow/step/status.rb
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
 - lib/rbbt/workflow/util/archive.rb