rbbt-util 5.44.1 → 6.0.3

data/lib/rbbt-util.rb

@@ -1,18 +1,26 @@
-require 'rbbt'
-require 'rbbt/util/misc'
-require 'rbbt/util/open'
-Open.cachedir        = Rbbt.var.cache["open-remote"].find :user
+require_relative 'rbbt'
+require_relative 'rbbt/util/misc'
+require 'scout/open'
+Open.remote_cache_dir = Rbbt.var.cache["open-remote"].find :user
 
-require 'rbbt/persist'
-Persist.cachedir = Rbbt.var.cache.persistence
+require 'scout/path'
 
-require 'rbbt/util/filecache'
-FileCache.cachedir   = Rbbt.var.cache.filecache.find :user
+require 'set'
+require 'scout/persist'
 
-require 'rbbt/util/tmpfile'
-TmpFile.tmpdir       = Rbbt.tmp.find :user
+require 'scout/resource'
 
-require 'rbbt/util/cmd'
-require 'rbbt/tsv'
+require_relative 'rbbt/util/filecache'
+
+require_relative 'rbbt/util/tmpfile'
+
+require_relative 'rbbt/util/cmd'
+require_relative 'rbbt/tsv'
+
+require_relative 'rbbt/workflow'
+
+Persist.cache_dir     = Rbbt.var.cache.persistence
+FileCache.cachedir    = Rbbt.var.cache.filecache.find :user
+TmpFile.tmpdir        = Rbbt.tmp.find :user
+Resource.default_resource = Rbbt
 
-require 'rbbt/util/config'

data/lib/rbbt.rb

@@ -1,7 +1,20 @@
-require 'rbbt/resource'
+$LOAD_PATH.unshift File.join(__dir__)
+$LOAD_PATH.unshift File.join(__dir__, '../../lib')
+
+require 'scout/path'
+require 'scout/resource'
+Path.add_path :rbbt_util, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
 module Rbbt
   extend Resource
-  pkgdir = 'rbbt'
+
+  self.pkgdir = 'rbbt'
 end
+Path.default_pkgdir   = Rbbt
+Resource.set_software_env Rbbt.software
+
+require_relative 'rbbt/util/version'
+require 'scout/log'
+require_relative 'rbbt/refactor'
 
-require 'rbbt/util/version'
+require 'scout/config'
+Rbbt::Config = Scout::Config

data/python/rbbt/__init__.py

@@ -3,7 +3,7 @@ import sys
 import os
 import subprocess
 
-def rbbt(cmd = None):
+def cmd(cmd = None):
     if cmd is None:
         print("Rbbt")
     else:
@@ -52,6 +52,15 @@ def log_tsv(tsv):
     print(tsv)
     print(tsv.keys())
 
+def benchmark():
+    import time
+    tic: float = time.perf_counter()
+    try:
+        yield
+    finally:
+        toc: float = time.perf_counter()
+    print(f"Computation time = {1000*(toc - tic):.3f}ms")
+
 def tsv_preamble(line, comment_char="#"):
     import re
     header = dict()
@@ -127,3 +136,12 @@ def tsv_pandas(filename, sep="\t", comment_char="#", index_col=0, **kwargs):
 def tsv(*args, **kwargs):
     return tsv_pandas(*args, **kwargs)
 
+def save_tsv(filename, df, key=None):
+    if (key == None):
+        key = df.index.name
+    if (key == None):
+        key = "Key"
+    key = "#" + key
+    df.to_csv(filename, sep="\t", index_label=key)
+
+

data/share/Rlib/plot.R

@@ -1,42 +1,42 @@
 rbbt.require('ggplot2')
 
-geom_entity <- function (real.geom = NULL, mapping = NULL, data = NULL, stat = "identity",
-                          position = "identity", ...) {
-    rg <- real.geom(mapping = mapping, data = data, stat = stat, 
-                    position = position, ...)
-
-    rg$geom <- proto(rg$geom, { 
-        draw <- function(., data, ...) {
-            grobs <- list()
-
-            for (i in 1:nrow(data)) {
-                grob <- .super$draw(., data[i,], ...) 
-                if (is.null(data$entity.type)) 
-                    grobs[[i]] <- garnishGrob(grob, `data-entity`=data[i,]$entity)
-                else
-                    grobs[[i]] <- garnishGrob(grob, `data-entity`=data[i,]$entity, `data-entity-type`=data[i,]$entity.type)
-            }
-
-            ggplot2:::ggname("geom_entity", gTree(children = do.call("gList", grobs)))
-        }
-
-        draw_groups <- function(., data, ...) {
-            grobs <- list()
-
-            for (i in 1:nrow(data)) {
-                grob <- .super$draw_groups(., data[i,], ...) 
-                if (is.null(data$entity.type)) 
-                    grobs[[i]] <- garnishGrob(grob, `data-entity`=data[i,]$entity)
-                else
-                    grobs[[i]] <- garnishGrob(grob, `data-entity`=data[i,]$entity, `data-entity-type`=data[i,]$entity.type)
-            }
-
-            ggplot2:::ggname("geom_entity", gTree(children = do.call("gList", grobs)))
-        }
-    })
-
-    rg
-}
+#geom_entity <- function (real.geom = NULL, mapping = NULL, data = NULL, stat = "identity",
+#                          position = "identity", ...) {
+#    rg <- real.geom(mapping = mapping, data = data, stat = stat, 
+#                    position = position, ...)
+#
+#    rg$geom <- ggproto(rg$geom, { 
+#        draw <- function(., data, ...) {
+#            grobs <- list()
+#
+#            for (i in 1:nrow(data)) {
+#                grob <- .super$draw(., data[i,], ...) 
+#                if (is.null(data$entity.type)) 
+#                    grobs[[i]] <- garnishGrob(grob, `data-entity`=data[i,]$entity)
+#                else
+#                    grobs[[i]] <- garnishGrob(grob, `data-entity`=data[i,]$entity, `data-entity-type`=data[i,]$entity.type)
+#            }
+#
+#            ggplot2:::ggname("geom_entity", gTree(children = do.call("gList", grobs)))
+#        }
+#
+#        draw_groups <- function(., data, ...) {
+#            grobs <- list()
+#
+#            for (i in 1:nrow(data)) {
+#                grob <- .super$draw_groups(., data[i,], ...) 
+#                if (is.null(data$entity.type)) 
+#                    grobs[[i]] <- garnishGrob(grob, `data-entity`=data[i,]$entity)
+#                else
+#                    grobs[[i]] <- garnishGrob(grob, `data-entity`=data[i,]$entity, `data-entity-type`=data[i,]$entity.type)
+#            }
+#
+#            ggplot2:::ggname("geom_entity", gTree(children = do.call("gList", grobs)))
+#        }
+#    })
+#
+#    rg
+#}
 
 rbbt.ggplot2.rotate_x_labels <- function(){ theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) }
 

data/share/Rlib/svg.R

@@ -9,12 +9,17 @@ rbbt.require('Cairo')
 
 # Modified from http://aaronecay.com/blog/2014/02/tooltips-in-ggplot/
 
-rbbt.SVG.extract <- function(plot, size=NULL, prefix=NULL, ...){
+rbbt.SVG.extract <- function(plot, size=NULL, prefix=NULL, entity.geom='geom_point', data=NULL, ...){
+
+    if (is.null(data)) data = plot$data;
 
     if (is.null(prefix)) prefix = rbbt.random_string();
+
+    if (!endsWith(prefix, '.'))
+        prefix = paste(prefix, ".", sep="")
+
     if (is.null(size)){
         print(plot, type='cairo');
-        mysvg <- grid.export(prefix=prefix, ...)
     }else{
         base.size = 10 * (7/size)
         resolution = 72 * (size/7)
@@ -25,11 +30,23 @@ rbbt.SVG.extract <- function(plot, size=NULL, prefix=NULL, ...){
 
         if (is.null(plot$theme$text$size))
           plot$theme$text$size = base.size
-
-        print(plot, type='cairo')
-        mysvg <- grid.export(res=resolution, prefix=prefix, ...)
+        print(plot, type='cairo');
     }
 
+    grid.force()
+
+    if (!is.null(data[["Entity"]]))
+        grid.garnish(entity.geom, 'data-entity'= data[["Entity"]], group = FALSE, grep = TRUE, redraw = TRUE)
+    else
+        grid.garnish(entity.geom, 'data-entity'= rownames(data), group = FALSE, grep = TRUE, redraw = TRUE)
+
+    if (!is.null(data[["Entity type"]]))
+        grid.garnish(entity.geom, 'data-entity_type' = data[["Entity type"]], group = FALSE, grep = TRUE, redraw = TRUE)
+    else if (!is.null(attributes(data)$key.field))
+        grid.garnish(entity.geom, 'data-entity_type' = rep(attributes(data)$key.field, length(rownames(data))), group = FALSE, grep = TRUE, redraw = TRUE)
+
+    mysvg <- grid.export(prefix=prefix, strict = FALSE, ...)
+
     xml <- saveXML(mysvg$svg)
     xml
 }

data/share/install/software/lib/install_helpers

@@ -10,7 +10,7 @@ OPT_BUILD_DIR="$SOFTWARE_DIR/.build"; [ -d $OPT_BUILD_DIR ] || mkdir -p $OPT_BUI
 
 mkdir -p "$OPT_BUILD_DIR"
 
-if realpath --help | grep realative > /dev/null; then
+if (command -v realpath && realpath --help | grep relative) > /dev/null; then
 	function relative_path(){
 		local basedir=$1
 		local path=$2

data/share/rbbt_commands/hpc/list

@@ -206,7 +206,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
         require 'rbbt/workflow'
         step_path = step_line.split(": ").last.strip
         step = Step.new step_path
-        step.load_dependencies_from_info
         has_bar = false
         [step].reverse.each do |j|
           next if j.done? 
@@ -218,6 +217,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
             prog_rep << [rep]
           end
         end
+        prog_rep = [Log.color(step.status, step.status)] if prog_rep.empty?
       end
     end
     workflow, task, name = step_path.split("/")[-3..-1]
@@ -321,7 +321,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   if tail && File.exist?(File.join(dir, 'std.err'))
     if exit_status && exit_status != 0
       puts Log.color(:magenta, "First error or exception found: ")
-      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+      puts CMD.cmd("grep -i 'error\\|exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
     elsif exit_status
       puts Log.color(:magenta, "Completed jobs: ")
       puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
@@ -337,7 +337,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       require 'rbbt/workflow'
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
-      step.load_dependencies_from_info
       has_bar = false
       (step.rec_dependencies + [step]).reverse.each do |j|
         next if j.done? 

data/share/rbbt_commands/hpc/orchestrate

@@ -18,9 +18,9 @@ $slurm_options = SOPT.get <<EOF
 -si--singularity_img* Singularity image to use
 -sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
+--contain* Contain in directory (using Singularity)
+--sync* Contain in directory and sync jobs
+--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
 -pd--purge_deps Purge job dependencies
@@ -57,7 +57,7 @@ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint
 
 class Step
   def join
-    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir) unless done?
   end
 
   def run(no_load = true)

data/share/rbbt_commands/hpc/tail

@@ -33,6 +33,8 @@ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint
 
 directory = ARGV.shift
 
+directory = HPC::BATCH_MODULE.jobs.first if directory.nil?
+
 raise ParameterException if directory.nil?
 
 if directory =~ /^[0-9]*$/

data/share/rbbt_commands/hpc/task

@@ -9,7 +9,7 @@ require 'time'
 
 rbbt_options = SOPT::GOT_OPTIONS
 
-$slurm_options = SOPT.get <<EOF
+$batch_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
 --drbbt* Use development version of rbbt 
@@ -17,9 +17,9 @@ $slurm_options = SOPT.get <<EOF
 -si--singularity_img* Singularity image to use
 -sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
+--contain* Contain in directory (using Singularity)
+--sync* Contain in directory and sync jobs
+--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
 -pd--purge_deps Purge job dependencies
@@ -45,11 +45,14 @@ $slurm_options = SOPT.get <<EOF
 -OR--orchestration_rules* Orchestration rules
 EOF
 
-batch_system = $slurm_options.delete :batch_system
+batch_system = $batch_options.delete :batch_system
 batch_system ||= 'auto'
+$batch_options[:log] = rbbt_options[:log]
+$batch_options[:config_keys] = rbbt_options[:config_keys]
 
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
+
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
@@ -60,7 +63,7 @@ class Step
     else
       begin
         Log.debug "Issuing BATCH job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
+        HPC::BATCH_MODULE.run_job(self, $batch_options) 
       rescue HPC::BATCH_DRY_RUN
       end
     end
@@ -70,7 +73,7 @@ end
 module RemoteStep::SSH
 
   def _run
-    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $batch_options)
   end
 
 end

data/share/rbbt_commands/lsf/list

@@ -206,7 +206,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
         require 'rbbt/workflow'
         step_path = step_line.split(": ").last.strip
         step = Step.new step_path
-        step.load_dependencies_from_info
         has_bar = false
         [step].reverse.each do |j|
           next if j.done? 
@@ -218,6 +217,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
             prog_rep << [rep]
           end
         end
+        prog_rep = [Log.color(step.status, step.status)] if prog_rep.empty?
       end
     end
     workflow, task, name = step_path.split("/")[-3..-1]
@@ -321,7 +321,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   if tail && File.exist?(File.join(dir, 'std.err'))
     if exit_status && exit_status != 0
       puts Log.color(:magenta, "First error or exception found: ")
-      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+      puts CMD.cmd("grep -i 'error\\|exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
     elsif exit_status
       puts Log.color(:magenta, "Completed jobs: ")
       puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
@@ -337,7 +337,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       require 'rbbt/workflow'
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
-      step.load_dependencies_from_info
       has_bar = false
       (step.rec_dependencies + [step]).reverse.each do |j|
         next if j.done? 

data/share/rbbt_commands/lsf/orchestrate

@@ -18,9 +18,9 @@ $slurm_options = SOPT.get <<EOF
 -si--singularity_img* Singularity image to use
 -sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
+--contain* Contain in directory (using Singularity)
+--sync* Contain in directory and sync jobs
+--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
 -pd--purge_deps Purge job dependencies
@@ -57,7 +57,7 @@ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint
 
 class Step
   def join
-    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir) unless done?
   end
 
   def run(no_load = true)

data/share/rbbt_commands/lsf/tail

@@ -33,6 +33,8 @@ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint
 
 directory = ARGV.shift
 
+directory = HPC::BATCH_MODULE.jobs.first if directory.nil?
+
 raise ParameterException if directory.nil?
 
 if directory =~ /^[0-9]*$/

data/share/rbbt_commands/lsf/task

@@ -9,7 +9,7 @@ require 'time'
 
 rbbt_options = SOPT::GOT_OPTIONS
 
-$slurm_options = SOPT.get <<EOF
+$batch_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
 --drbbt* Use development version of rbbt 
@@ -17,9 +17,9 @@ $slurm_options = SOPT.get <<EOF
 -si--singularity_img* Singularity image to use
 -sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
+--contain* Contain in directory (using Singularity)
+--sync* Contain in directory and sync jobs
+--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
 -pd--purge_deps Purge job dependencies
@@ -45,11 +45,14 @@ $slurm_options = SOPT.get <<EOF
 -OR--orchestration_rules* Orchestration rules
 EOF
 
-batch_system = $slurm_options.delete :batch_system
+batch_system = $batch_options.delete :batch_system
 batch_system ||= 'auto'
+$batch_options[:log] = rbbt_options[:log]
+$batch_options[:config_keys] = rbbt_options[:config_keys]
 
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
+
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
@@ -60,7 +63,7 @@ class Step
     else
       begin
         Log.debug "Issuing BATCH job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
+        HPC::BATCH_MODULE.run_job(self, $batch_options) 
       rescue HPC::BATCH_DRY_RUN
       end
     end
@@ -70,7 +73,7 @@ end
 module RemoteStep::SSH
 
   def _run
-    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $batch_options)
   end
 
 end

data/share/rbbt_commands/migrate

@@ -36,4 +36,4 @@ search_path = 'user' if search_path.nil?
 
 options[:other] = other
 
-Rbbt.migrate(path, search_path, options)
+Rbbt.migrate(path.dup, search_path, options)

data/share/rbbt_commands/pbs/list

@@ -206,7 +206,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
         require 'rbbt/workflow'
         step_path = step_line.split(": ").last.strip
         step = Step.new step_path
-        step.load_dependencies_from_info
         has_bar = false
         [step].reverse.each do |j|
           next if j.done? 
@@ -218,6 +217,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
             prog_rep << [rep]
           end
         end
+        prog_rep = [Log.color(step.status, step.status)] if prog_rep.empty?
       end
     end
     workflow, task, name = step_path.split("/")[-3..-1]
@@ -321,7 +321,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   if tail && File.exist?(File.join(dir, 'std.err'))
     if exit_status && exit_status != 0
       puts Log.color(:magenta, "First error or exception found: ")
-      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+      puts CMD.cmd("grep -i 'error\\|exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
     elsif exit_status
       puts Log.color(:magenta, "Completed jobs: ")
       puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
@@ -337,7 +337,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       require 'rbbt/workflow'
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
-      step.load_dependencies_from_info
       has_bar = false
       (step.rec_dependencies + [step]).reverse.each do |j|
         next if j.done? 

data/share/rbbt_commands/pbs/orchestrate

@@ -18,9 +18,9 @@ $slurm_options = SOPT.get <<EOF
 -si--singularity_img* Singularity image to use
 -sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
+--contain* Contain in directory (using Singularity)
+--sync* Contain in directory and sync jobs
+--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
 -pd--purge_deps Purge job dependencies
@@ -57,7 +57,7 @@ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint
 
 class Step
   def join
-    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir) unless done?
   end
 
   def run(no_load = true)

data/share/rbbt_commands/pbs/tail

@@ -33,6 +33,8 @@ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint
 
 directory = ARGV.shift
 
+directory = HPC::BATCH_MODULE.jobs.first if directory.nil?
+
 raise ParameterException if directory.nil?
 
 if directory =~ /^[0-9]*$/

data/share/rbbt_commands/pbs/task

@@ -9,7 +9,7 @@ require 'time'
 
 rbbt_options = SOPT::GOT_OPTIONS
 
-$slurm_options = SOPT.get <<EOF
+$batch_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
 --drbbt* Use development version of rbbt 
@@ -17,9 +17,9 @@ $slurm_options = SOPT.get <<EOF
 -si--singularity_img* Singularity image to use
 -sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
+--contain* Contain in directory (using Singularity)
+--sync* Contain in directory and sync jobs
+--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
 -pd--purge_deps Purge job dependencies
@@ -45,11 +45,14 @@ $slurm_options = SOPT.get <<EOF
 -OR--orchestration_rules* Orchestration rules
 EOF
 
-batch_system = $slurm_options.delete :batch_system
+batch_system = $batch_options.delete :batch_system
 batch_system ||= 'auto'
+$batch_options[:log] = rbbt_options[:log]
+$batch_options[:config_keys] = rbbt_options[:config_keys]
 
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
+
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
@@ -60,7 +63,7 @@ class Step
     else
       begin
         Log.debug "Issuing BATCH job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
+        HPC::BATCH_MODULE.run_job(self, $batch_options) 
       rescue HPC::BATCH_DRY_RUN
       end
     end
@@ -70,7 +73,7 @@ end
 module RemoteStep::SSH
 
   def _run
-    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $batch_options)
   end
 
 end

data/share/rbbt_commands/resource/produce

@@ -54,5 +54,12 @@ end
 
 force = options[:force]
 
-puts resource[path].produce(force).find
+file = resource[path].produce(force)
+
+if file
+    puts file.find
+else
+    puts Log.color("Error: Could not produce #{path} (#{resource})")
+    exit -1
+end
 

data/share/rbbt_commands/slurm/list

@@ -206,7 +206,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
         require 'rbbt/workflow'
         step_path = step_line.split(": ").last.strip
         step = Step.new step_path
-        step.load_dependencies_from_info
         has_bar = false
         [step].reverse.each do |j|
           next if j.done? 
@@ -218,6 +217,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
             prog_rep << [rep]
           end
         end
+        prog_rep = [Log.color(step.status, step.status)] if prog_rep.empty?
       end
     end
     workflow, task, name = step_path.split("/")[-3..-1]
@@ -321,7 +321,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   if tail && File.exist?(File.join(dir, 'std.err'))
     if exit_status && exit_status != 0
       puts Log.color(:magenta, "First error or exception found: ")
-      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+      puts CMD.cmd("grep -i 'error\\|exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
     elsif exit_status
       puts Log.color(:magenta, "Completed jobs: ")
       puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
@@ -337,7 +337,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       require 'rbbt/workflow'
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
-      step.load_dependencies_from_info
       has_bar = false
       (step.rec_dependencies + [step]).reverse.each do |j|
         next if j.done?