rbbt-util 5.32.12 → 5.32.17

data/lib/rbbt/workflow/usage.rb

@@ -67,11 +67,13 @@ end
 
 module Workflow
 
-  def dep_tree(name)
+  def dep_tree(name, seen = [])
     @dep_tree ||= {}
     @dep_tree[name] ||= begin
                         dep_tree = {}
                         self.task_dependencies[name.to_sym].reverse.each do |dep|
+                          next if seen.include? dep
+                          seen << dep
                           dep = dep.first if Array === dep && dep.length == 1
                           dep = dep.dependency if DependencyBlock === dep
 
@@ -87,7 +89,7 @@ module Workflow
 
                           key = [workflow, task]
 
-                          dep_tree[key] = workflow.dep_tree(task)
+                          dep_tree[key] = workflow.dep_tree(task, seen)
                         end if name && self.task_dependencies[name.to_sym]
                         dep_tree
                       end

data/lib/rbbt/workflow/util/data.rb

@@ -0,0 +1,35 @@
+require 'rbbt/workflow'
+require 'rbbt/workflow/examples'
+
+module Workflow
+  module Data
+    def data(directory)
+      @@data_directory = directory
+    end
+
+    def get_datadir(clean_name)
+      data_dir = File.join(@@data_directory, clean_name)
+      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+      if Path === @@data_directory
+        @@data_directory.annotate data_dir
+      else
+        Path.setup(data_dir)
+      end
+    end
+
+    def data_task(name, workflow, oname, *rest, &block)
+      dep_task(name, workflow, oname, *rest) do |jobname, options|
+        data_dir = self.get_datadir(jobname)
+        task_info = workflow.task_info(oname)
+        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
+        data_options = block.call data_dir, dir_options, task_info
+        if data_options.include?(:inputs)
+          data_options
+        else
+          {:inputs => data_options.merge(options)}
+        end
+      end
+    end
+  end
+
+end

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -81,7 +81,7 @@ module Workflow
     def self.candidates(workload, rules)
       if rules.empty?
         candidates = workload.select{|k,v| v.empty? }.
-          collect{|k,v| k}.
+          collect{|k,v| k }.
           reject{|k| k.done? }
       else
         candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.

data/lib/rbbt/workflow/util/provenance.rb

@@ -26,7 +26,7 @@ class Step
     Log.color(color, status.to_s)
   end
 
-  def self.prov_report_msg(status, name, path, info = nil)
+  def self.prov_report_msg(status, name, path, info, input = nil)
     parts = path.sub(/\{.*/,'').split "/"
 
     parts.pop
@@ -43,10 +43,19 @@ class Step
                  rescue Exception
                    nil
                  end
+
+    if input.nil? || input.empty?
+      input_str = nil
+    else
+      input = input.reject{|dep,name| (input & dep.dependencies.collect{|d| [d,name]}).any? }
+      input = input.reject{|dep,name| (input & dep.input_dependencies.collect{|d| [d,name]}).any? }
+      input_str = Log.color(:magenta, "-> ") + input.collect{|dep,name| Log.color(:yellow, dep.task_name.to_s) + ":" + Log.color(:yellow, name) }.uniq * " "
+    end
+
     str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
-            prov_status_msg(status.to_s) << " " << [workflow, task, path].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
           else
-            prov_status_msg(status.to_s) << " " << [workflow, task, path].compact * " " 
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " 
           end
 
     if $inputs and $inputs.any? 
@@ -74,7 +83,7 @@ class Step
     str << "\n"
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false)
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -84,14 +93,33 @@ class Step
     status = :unsync if status == :done and not Open.exist?(path)
     status = :notfound if status == :noinfo and not Open.exist?(path)
 
-    str = " " * offset
-    str << prov_report_msg(status, name, path, info)
-    step.dependencies.reverse.each do |dep|
+
+    this_step_msg = prov_report_msg(status, name, path, info, input)
+
+    input_dependencies = {}
+    step.dependencies.each do |dep|
+      if dep.input_dependencies.any?
+        dep.input_dependencies.each do |id|
+          input_name = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+            d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
+          }.first.first
+          input_dependencies[id] ||= []
+          input_dependencies[id] << [dep, input_name]
+        end
+      end
+    end
+
+    str = ""
+    str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
+    step.dependencies.dup.tap{|l| 
+      l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    }.each do |dep|
       path = dep.path
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen, expand_repeats)
+        str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
       else
         if expand_repeats
           str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
@@ -103,10 +131,13 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
         end
       end
     end if step.dependencies
+
+    str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
     str
   end
 end

data/python/rbbt.py

@@ -0,0 +1,7 @@
+import warnings
+import sys
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+def rbbt():
+    print("Rbbt")

data/share/install/software/lib/install_helpers

@@ -499,7 +499,7 @@ install_jar(){
     local url="$2"
 
     [ -d  "$OPT_DIR/$name/" ] || mkdir -p  "$OPT_DIR/$name/"
-    wget "$url" -O "$OPT_DIR/jars/$name.jar" || wget "$url" -O "$OPT_DIR/jars/$name.jar" --no-check-certificate || (rm "$OPT_DIR/jars/$name.jar"; exit -1)
+    wget "$url" -O "$OPT_DIR/$name/$name.jar" || wget "$url" -O "$OPT_DIR/$name/$name.jar" --no-check-certificate || (rm "$OPT_DIR/$name/$name.jar"; exit -1)
     link "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar" 
 }
 

data/share/rbbt_commands/hpc/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/hpc/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/lsf/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/lsf/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/slurm/orchestrate

@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/slurm/task

@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
 -mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/system/clean

@@ -34,6 +34,7 @@ force = options.delete :force
 dirty = options.delete :dirty
 time = options.delete :older
 
+ENV["RBBT_DEBUG_CLEAN"] = 'false'
 ENV["RBBT_UPDATE"] = 'true' if dirty
 
 time = Misc.timespan time, 'd' if time

data/share/rbbt_commands/workflow/prov

@@ -131,6 +131,6 @@ if options[:plot]
   end
 
 else
-  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats]).strip
+  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats])
 end
 

data/test/rbbt/util/test_python.rb

@@ -2,6 +2,7 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
 require 'rbbt/util/python'
 
 class TestPython < Test::Unit::TestCase
+
   def test_python
     TmpFile.with_file do |tmpdir|
       code =<<-EOF
@@ -67,7 +68,7 @@ def python_print():
 
   def test_keras
     defined = RbbtPython.run do
-      pyimport "keras.models", as: :km
+      pyimport "tensorflow.keras.models", as: :km
       defined?(km.Sequential)
     end
     assert defined
@@ -75,7 +76,7 @@ def python_print():
 
   def test_keras_import
     defined = RbbtPython.run do
-      pyfrom "keras.models", import: :Sequential
+      pyfrom "tensorflow.keras.models", import: :Sequential
       defined?(self::Sequential)
     end
     assert defined

data/test/rbbt/workflow/util/test_data.rb

@@ -0,0 +1,48 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow/util/data'
+
+module TestDataWF
+  extend Workflow
+  extend Workflow::Data
+
+  input :salutation, :string
+  task :salute_luis => :string do |name,salutation|
+    "Hi Luis: #{salutation}"
+  end
+
+  input :name, :string
+  input :salutation, :string
+  task :salute => :string do |name,salutation|
+    "Hi #{name}: #{salutation}"
+  end
+
+  data_task :salute_data, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
+    options.merge({:salutation => directory.salutation.read})
+  end
+
+  data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
+    {:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
+  end
+end
+
+class TestWorkflowData < Test::Unit::TestCase
+  def test_workflow_data
+    TmpFile.with_file do |tmpdir|
+      tmpdir = Path.setup(tmpdir.dup)
+
+      Open.write(tmpdir.TestDir.options.name, "Miguel")
+      Open.write(tmpdir.TestDir.salutation, "My salutations")
+
+      TestDataWF.data tmpdir
+
+      job = TestDataWF.job(:salute_data, "TestDir")
+      job.recursive_clean.run
+      assert job.run.include? "Miguel"
+
+      job = TestDataWF.job(:salute_data2, "TestDir")
+      job.recursive_clean.run
+      assert job.run.include? "Luis"
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.12
+  version: 5.32.17
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-25 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -322,6 +322,7 @@ files:
 - lib/rbbt/workflow/accessor.rb
 - lib/rbbt/workflow/annotate.rb
 - lib/rbbt/workflow/definition.rb
+- lib/rbbt/workflow/dependencies.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
 - lib/rbbt/workflow/integration/ansible.rb
@@ -339,14 +340,18 @@ files:
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/accessor.rb
 - lib/rbbt/workflow/step/dependencies.rb
+- lib/rbbt/workflow/step/info.rb
 - lib/rbbt/workflow/step/prepare.rb
 - lib/rbbt/workflow/step/run.rb
+- lib/rbbt/workflow/step/status.rb
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
 - lib/rbbt/workflow/util/archive.rb
+- lib/rbbt/workflow/util/data.rb
 - lib/rbbt/workflow/util/orchestrator.rb
 - lib/rbbt/workflow/util/provenance.rb
 - lib/rbbt/workflow/util/trace.rb
+- python/rbbt.py
 - share/Rlib/plot.R
 - share/Rlib/svg.R
 - share/Rlib/util.R
@@ -549,6 +554,7 @@ files:
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/workflow/util/test_archive.rb
+- test/rbbt/workflow/util/test_data.rb
 - test/rbbt/workflow/util/test_orchestrator.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-util
@@ -579,6 +585,7 @@ test_files:
 - test/rbbt/workflow/test_remote_workflow.rb
 - test/rbbt/workflow/util/test_archive.rb
 - test/rbbt/workflow/util/test_orchestrator.rb
+- test/rbbt/workflow/util/test_data.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb