rbbt-util 5.32.10 → 5.32.16

data/lib/rbbt/workflow/util/data.rb

@@ -0,0 +1,35 @@
+require 'rbbt/workflow'
+require 'rbbt/workflow/examples'
+
+module Workflow
+  module Data
+    def data(directory)
+      @@data_directory = directory
+    end
+
+    def get_datadir(clean_name)
+      data_dir = File.join(@@data_directory, clean_name)
+      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+      if Path === @@data_directory
+        @@data_directory.annotate data_dir
+      else
+        Path.setup(data_dir)
+      end
+    end
+
+    def data_task(name, workflow, oname, *rest, &block)
+      dep_task(name, workflow, oname, *rest) do |jobname, options|
+        data_dir = self.get_datadir(jobname)
+        task_info = workflow.task_info(oname)
+        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
+        data_options = block.call data_dir, dir_options, task_info
+        if data_options.include?(:inputs)
+          data_options
+        else
+          {:inputs => data_options.merge(options)}
+        end
+      end
+    end
+  end
+
+end

data/lib/rbbt/workflow/util/provenance.rb

@@ -26,7 +26,7 @@ class Step
     Log.color(color, status.to_s)
   end
 
-  def self.prov_report_msg(status, name, path, info = nil)
+  def self.prov_report_msg(status, name, path, info, input = nil)
     parts = path.sub(/\{.*/,'').split "/"
 
     parts.pop
@@ -43,10 +43,17 @@ class Step
                  rescue Exception
                    nil
                  end
+
+    if input.nil? || input.empty?
+      input_str = nil
+    else
+      input_str = Log.color(:magenta, "-> ") + input.collect{|dep,name| Log.color(:yellow, dep.task_name.to_s) + ":" + Log.color(:yellow, name) }.uniq * " "
+    end
+
     str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
-            prov_status_msg(status.to_s) << " " << [workflow, task, path].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
           else
-            prov_status_msg(status.to_s) << " " << [workflow, task, path].compact * " " 
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " 
           end
 
     if $inputs and $inputs.any? 
@@ -74,7 +81,7 @@ class Step
     str << "\n"
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false)
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -85,13 +92,25 @@ class Step
     status = :notfound if status == :noinfo and not Open.exist?(path)
 
     str = " " * offset
-    str << prov_report_msg(status, name, path, info)
+    str << prov_report_msg(status, name, path, info, input)
+
+    input_dependencies = {}
+    step.dependencies.each do |dep|
+      if dep.input_dependencies.any?
+        dep.input_dependencies.each do |id|
+          input_name = dep.inputs.fields.zip(dep.inputs).select{|f,d| d == id || String === d && d.start_with?(id.files_dir) }.first.first
+          input_dependencies[id] ||= []
+          input_dependencies[id] << [dep, input_name]
+        end
+      end
+    end
+
     step.dependencies.reverse.each do |dep|
       path = dep.path
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen, expand_repeats)
+        str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
       else
         if expand_repeats
           str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
@@ -103,7 +122,7 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
         end
       end
     end if step.dependencies

data/lib/rbbt/workflow/util/trace.rb

@@ -188,6 +188,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
     seed_jobs.each do |step|
       jobs += step.rec_dependencies + [step]
       step.info[:archived_info].each do |path,ainfo|
+        next unless Hash === ainfo
         archived_step = Step.new path
 
         archived_step.define_singleton_method :info do
@@ -205,7 +206,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
 
     end
 
-    jobs = jobs.uniq.sort_by{|job| t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
+    jobs = jobs.uniq.sort_by{|job| [job, job.info]; t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
 
     data = trace_job_times(jobs, options[:fix_gap])
 

data/python/rbbt.py

@@ -0,0 +1,7 @@
+import warnings
+import sys
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+def rbbt():
+    print("Rbbt")

data/share/install/software/lib/install_helpers

@@ -499,7 +499,7 @@ install_jar(){
     local url="$2"
 
     [ -d  "$OPT_DIR/$name/" ] || mkdir -p  "$OPT_DIR/$name/"
-    wget "$url" -O "$OPT_DIR/jars/$name.jar" || wget "$url" -O "$OPT_DIR/jars/$name.jar" --no-check-certificate || (rm "$OPT_DIR/jars/$name.jar"; exit -1)
+    wget "$url" -O "$OPT_DIR/$name/$name.jar" || wget "$url" -O "$OPT_DIR/$name/$name.jar" --no-check-certificate || (rm "$OPT_DIR/$name/$name.jar"; exit -1)
     link "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar" 
 }
 

data/share/rbbt_commands/hpc/list

@@ -173,7 +173,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
-   select = select && cmd.match(/#{search}/) if search
+   select = select && step_path.match(/#{search}/) if search
    next unless select 
  elsif search
    select = false
@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end
@@ -292,13 +292,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      step_path = step_line.split(": ").last.strip
      step = Step.new step_path
      step.load_dependencies_from_info
+     has_bar = false
      (step.rec_dependencies + [step]).reverse.each do |j|
        next if j.done? 
-       next unless j.file(:progress).exists?
-       bar = Log::ProgressBar.new 
-       bar.load(j.file(:progress).yaml)
-       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+       if j.file(:progress).exists?
+         bar = Log::ProgressBar.new 
+         bar.load(j.file(:progress).yaml)
+         puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+         has_bar = true
+       end
      end
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
    end
  end
 

data/share/rbbt_commands/hpc/orchestrate

@@ -19,6 +19,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -26,11 +27,16 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/hpc/task

@@ -18,6 +18,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -25,10 +26,14 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/lsf/list

@@ -173,7 +173,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
-   select = select && cmd.match(/#{search}/) if search
+   select = select && step_path.match(/#{search}/) if search
    next unless select 
  elsif search
    select = false
@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end
@@ -292,13 +292,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      step_path = step_line.split(": ").last.strip
      step = Step.new step_path
      step.load_dependencies_from_info
+     has_bar = false
      (step.rec_dependencies + [step]).reverse.each do |j|
        next if j.done? 
-       next unless j.file(:progress).exists?
-       bar = Log::ProgressBar.new 
-       bar.load(j.file(:progress).yaml)
-       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+       if j.file(:progress).exists?
+         bar = Log::ProgressBar.new 
+         bar.load(j.file(:progress).yaml)
+         puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+         has_bar = true
+       end
      end
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
    end
  end
 

data/share/rbbt_commands/lsf/orchestrate

@@ -19,6 +19,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -26,11 +27,16 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/lsf/task

@@ -18,6 +18,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -25,10 +26,14 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/slurm/list

@@ -173,7 +173,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
-   select = select && cmd.match(/#{search}/) if search
+   select = select && step_path.match(/#{search}/) if search
    next unless select 
  elsif search
    select = false
@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end
@@ -292,13 +292,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      step_path = step_line.split(": ").last.strip
      step = Step.new step_path
      step.load_dependencies_from_info
+     has_bar = false
      (step.rec_dependencies + [step]).reverse.each do |j|
        next if j.done? 
-       next unless j.file(:progress).exists?
-       bar = Log::ProgressBar.new 
-       bar.load(j.file(:progress).yaml)
-       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+       if j.file(:progress).exists?
+         bar = Log::ProgressBar.new 
+         bar.load(j.file(:progress).yaml)
+         puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+         has_bar = true
+       end
      end
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
    end
  end
 

data/share/rbbt_commands/slurm/orchestrate

@@ -19,6 +19,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -26,11 +27,16 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
--OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/slurm/task

@@ -18,6 +18,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -25,10 +26,14 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 

data/share/rbbt_commands/workflow/forget_deps

@@ -38,11 +38,11 @@ step.copy_files_dir
 dependencies = step.dependencies
 
 if remove && remove.any?
- remove_paths = dependencies.select do |dep|
-  remove.include?(dep.task_name) || remove.include?([dep.workflow.to_s, dep.task_name] * "#")
- end.collect{|dep| dep.path } 
+  remove_paths = dependencies.select do |dep|
+    remove.include?(dep.task_name) || remove.include?([dep.workflow.to_s, dep.task_name] * "#")
+  end.collect{|dep| dep.path } 
 else
- remove_paths = dependencies.collect{|dep| dep.path }
+  remove_paths = dependencies.collect{|dep| dep.path }
 end
 
 step.set_info :dependencies, step.info[:dependencies].reject{|info| remove_paths.include? info.last}
@@ -50,6 +50,7 @@ step.set_info :dependencies, step.info[:dependencies].reject{|info| remove_paths
 if options[:purge] || options[:recursive_purge]
   dependencies.each do |dependency|
     next unless remove_paths.include? dependency.path
+    next unless Open.exists?(dependency.info_file)
     Step.purge(dependency.path, options[:recursive_purge])
   end
 end