rbbt-util 5.29.2 → 5.29.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa648dbd0465a8c6a9a62820dc08a7f726dcd45cb541e7ef1c5e06b0e8343680
-  data.tar.gz: f05f659bfcfe3eef10a1d4ba786d40a7f835a08711ca23af9b5b619ddd61eeda
+  metadata.gz: 3ec6302ccfe3f38a074f7f0d10511090c8f4db4186228ad93adb2888e0edbf5e
+  data.tar.gz: fcaa50b654461f128b9539fc47ed00b008d3f713e89b8bbe963c2b898c3c168b
 SHA512:
-  metadata.gz: b44a34d7ad19eff67c7f8e6f3bfe0f55372e3d8a255247ff01d1aa858a904d2fcf73d45dd966702adf62bdbc3cb1b0dc3df643aaab747a922bfe3b747f975950
-  data.tar.gz: 5e74bcc37585452e22d876211f0c02c8fc431dbbdcc8cc0d3ef537ea794b722e0af9fc4ae743a014a1384c432f8f280761c74e35a1a8aaf25150067675a9f449
+  metadata.gz: 2a2537aef150df77142a593742399d28bb96334decc0e33b69e9cbac2853085487100aa90cd711b342bac7eda536d15c080be22891d5b9f38bfa4601282ae5de
+  data.tar.gz: d79cc4afa294d63cebd79f73b759bc46c3c4e0285db7be7406a594aa49e5b572b0c4f91b98c73b5f64c31c908d58d8dee43853ec18b1ead9e814b4370de0d60d

data/lib/rbbt/hpc/orchestrate.rb

@@ -60,9 +60,14 @@ module HPC
       return if job.done?
       return unless job.path.split("/")[-4] == "jobs"
       seen[:orchestration_target_job] ||= job
+
       options.delete "recursive_clean"
+      options.delete "clean_task"
+      options.delete "clean"
       options.delete "tail"
       options.delete "printfile"
+      options.delete "detach"
+
       rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
       rules ||= {}
       IndiferentHash.setup(rules)
@@ -75,7 +80,13 @@ module HPC
         skip_dep = job_rules["chain_tasks"] &&
           job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
           job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
-        seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen)
+
+        deps = seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen)
+        if job.canfail_paths.include? dep.path
+          [deps].flatten.collect{|id| ['canfail', id] * ":"}
+        else
+          deps
+        end
       end.flatten.compact.uniq
 
       skip = true if job_rules[:skip]

data/lib/rbbt/hpc/slurm.rb

@@ -58,11 +58,11 @@ module HPC
         when FalseClass 
           '--' << o << "=false"
         else
-          ['--' << o, "'#{v}'"] * " "
+          ['--' << o, "'#{v.to_s.gsub("'", '\'')}'"] * " "
         end
       end * " "
 
-      rbbt_cmd << " --config_keys='#{config_keys}'" if config_keys and not config_keys.empty?
+      rbbt_cmd << " --config_keys='#{config_keys.gsub("'", '\'')}'" if config_keys and not config_keys.empty?
 
       time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
@@ -76,6 +76,7 @@ module HPC
       fjob = File.join(slurm_basedir, 'job.id')
       fexit = File.join(slurm_basedir, 'exit.status')
       fsync = File.join(slurm_basedir, 'sync.log')
+      fsyncexit = File.join(slurm_basedir, 'sync.status')
       fcmd = File.join(slurm_basedir, 'command.slurm')
 
       #{{{ GENERATE TEMPLATE
@@ -107,10 +108,6 @@ module HPC
         EOF
       end
 
-      header +=<<-EOF
-#CMD: #{rbbt_cmd}
-      EOF
-
       # ENV
       env = ""
       env +=<<-EOF
@@ -246,7 +243,7 @@ EOF
         end
 
         if contain
-          rbbt_cmd << " " << %(--workdir_all='#{contain}')
+          rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/.rbbt/var/jobs')
         end
       end
 
@@ -256,6 +253,10 @@ EOF
 #{rbbt_cmd}
 EOF
 
+      header +=<<-EOF
+#CMD: #{rbbt_cmd}
+      EOF
+
       run +=<<-EOF
 
 # Run command
@@ -273,10 +274,10 @@ EOF
           coda +=<<-EOF
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean all -q &>> #{fsync}
 EOF
-        else
-          coda +=<<-EOF
-rbbt system clean all -q &>> #{fsync}
-EOF
+#        else
+#          coda +=<<-EOF
+#rbbt system clean all -q &>> #{fsync}
+#EOF
         end
 
         if sync.include?("=>")
@@ -295,6 +296,7 @@ EOF
 mkdir -p "$(dirname '#{target}')"
 rsync -avztAXHP --copy-unsafe-links "#{source}/" "#{target}/" &>> #{fsync} 
 sync_es="$?" 
+echo $sync_es > #{fsyncexit}
 find '#{target}' -type l -ls | awk '$13 ~ /^#{target.gsub('/','\/')}/ { sub("#{source}", "#{target}", $13); print $11, $13 }' | while read A B; do rm $A; ln -s $B $A; done
 EOF
 
@@ -320,23 +322,24 @@ singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -v /dev/shm/sem
 EOF
           else
             coda +=<<-EOF
-#{exec_cmd} system clean
+##{exec_cmd} system clean
 if [ $exit_status == '0' -a $sync_es == '0' ]; then 
     rm -Rfv #{contain} &>> #{fsync}
 else
     echo "ERROR: Process failed or results could not sync correctly. Contain directory not purged" &>> #{fsync}
 fi
-unset sync_es
 EOF
 
           end
         end
       end
+
       coda +=<<-EOF
 
 # Write exit status to file
 echo $exit_status > #{fexit}
 EOF
+
       if sync
         coda +=<<-EOF 
 if [ "$sync_es" == '0' ]; then 
@@ -362,6 +365,10 @@ EOF
       slurm_basedir = options[:slurm_basedir]
       dependencies = options.delete :slurm_dependencies
       dependencies = [] if dependencies.nil?
+
+      canfail_dependencies = dependencies.select{|dep| dep =~ /^canfail:(\d+)/ }.collect{|dep| dep.partition(":").last}
+      dependencies = dependencies.reject{|dep| dep =~ /^canfail:(\d+)/ }
+
       Open.mkdir slurm_basedir
 
       dry_run = options.delete :dry_run
@@ -370,6 +377,7 @@ EOF
       ferr  = File.join(slurm_basedir, 'std.err')
       fjob  = File.join(slurm_basedir, 'job.id')
       fdep  = File.join(slurm_basedir, 'dependencies.list')
+      fcfdep  = File.join(slurm_basedir, 'canfail_dependencies.list')
       fexit = File.join(slurm_basedir, 'exit.status')
       fsync = File.join(slurm_basedir, 'sync.log')
       fcmd  = File.join(slurm_basedir, 'command.slurm')
@@ -401,8 +409,21 @@ EOF
           Open.rm fexit
           Open.rm fout
           Open.rm ferr
+
           Open.write(fdep, dependencies * "\n") if dependencies.any?
-          dep_str = dependencies.any? ? "--dependency=afterok:" + dependencies * ":" : ''
+          Open.write(fcfdep, canfail_dependencies * "\n") if canfail_dependencies.any?
+
+
+          dep_str = '--dependency='
+          normal_dep_str = dependencies.any? ? "afterok:" + dependencies * ":" : nil
+          canfail_dep_str = canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
+
+          if normal_dep_str.nil? && canfail_dep_str.nil?
+            dep_str = ""
+          else
+            dep_str += [normal_dep_str, canfail_dep_str].compact * ","
+          end
+
           job = CMD.cmd("sbatch #{dep_str} '#{fcmd}'").read.scan(/\d+/).first.to_i
           Log.debug "SBATCH job id: #{job}"
           Open.write(fjob, job.to_s)
@@ -527,7 +548,7 @@ EOF
           cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--log', (options[:log] || Log.severity).to_s]
         end
 
-        cmd << "--override_deps='#{override_deps}'" if override_deps and not override_deps.empty?
+        cmd << "--override_deps='#{override_deps.gsub("'", '\'')}'" if override_deps and not override_deps.empty?
 
         template = self.template(cmd, options)
         jobid = self.issue_template(template, options.merge(:slurm_basedir => slurm_basedir, :dry_run => dry_run, :slurm_dependencies => dependencies))

data/lib/rbbt/persist.rb

@@ -110,6 +110,8 @@ module Persist
   def self.load_file(path, type)
     begin
       case (type || :marshal).to_sym
+      when :path
+        path
       when :nil
         nil
       when :boolean
@@ -167,6 +169,8 @@ module Persist
     end
 
     case (type || :marshal).to_sym
+    when :path
+      nil
     when :nil
       nil
     when :boolean

data/share/rbbt_commands/slurm/clean

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean error or aborted jobs
+
+$ rbbt mnl [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-SBP--sbatch_parameters show sbatch parameters
+-dr--dry_run Do not erase anything
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+Log.severity = 4
+done, error, aborted, jobid, search, tail, sbatch_parameters, dry_run = options.values_at :done, :error, :aborted, :job, :search, :tail, :sbatch_parameters, :dry_run
+
+workdir = File.expand_path('~/rbbt-slurm')
+Path.setup(workdir)
+
+running_jobs = begin
+                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
+               end
+
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
+count = 0
+workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+ dir = File.dirname(fcmd)
+
+ if m = Open.read(fcmd).match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+  exe = m[1]
+ else
+  exe = nil
+ end
+
+ if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+
+ if File.exists?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'job.status'))
+  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exists?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+
+ aborted = error = true if aborted.nil? && error.nil?
+ if done || error || aborted || running || queued || jobid || search
+   select = false
+   select = true if done && exit_status && exit_status.to_i == 0 
+   select = true if error && exit_status && exit_status.to_i != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   select = select && jobid.split(",").include?(id) if jobid 
+   select = select && cmd.match(/#{search}/) if search
+   next unless select 
+ end
+
+
+ puts Log.color(:yellow, "**ERASING**")
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:sbatch_parameters]
+   puts Log.color(:magenta, "SBATCH parameters: ")
+   puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ end
+
+ if tail && File.exists?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
+
+ Open.rm_rf dir unless dry_run
+end
+
+puts 
+puts "Found #{count} jobs"
+

data/share/rbbt_commands/slurm/list

@@ -20,15 +20,16 @@ $ rbbt mnl [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-SBP--sbatch_parameters show sbatch parameters
 EOF
 
 if options[:help]
-  if defined? rbbt_usage
-    rbbt_usage 
-  else
-    puts SOPT.doc
-  end
-  exit 0
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
 end
 
 Log.severity = 4
@@ -38,101 +39,124 @@ workdir = File.expand_path('~/rbbt-slurm')
 Path.setup(workdir)
 
 running_jobs = begin
-                   CMD.cmd('squeue').read.split("\n").collect{|l| l.to_i.to_s}
+                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
                rescue
-                   Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
-                   $norunningjobs = true
-                   []
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
                end
 
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
 count = 0
 workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
-    dir = File.dirname(fcmd)
-
-    if m = Open.read(fcmd).match(/#CMD: (.*)/)
-        cmd = m[1]
-    else
-        cmd = nil
-    end
-
-    if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
-        exe = m[1]
-    else
-        exe = nil
-    end
-
-    if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
-        container_home = m[1]
-    else
-        container_home = nil
-    end
-
-
-    if File.exists?(fid = File.join(dir, 'job.id'))
-        id = Open.read(fid).chomp
-    else
-        id = nil
-    end
-
-    if File.exists?(fstatus = File.join(dir, 'exit.status'))
-        exit_status = Open.read(fstatus).to_i
-    else
-        exit_status = nil
-    end
-    
-    if File.exists?(fstatus = File.join(dir, 'job.status'))
-        nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
-    else
-        nodes = []
-    end
-
-    if File.exists?(File.join(dir, 'std.out'))
-        outt = File.mtime File.join(dir, 'std.out')
-        errt = File.mtime File.join(dir, 'std.err')
-        time_diff = Time.now - [outt, errt].max
-    end
-
-    fdep = File.join(dir, 'dependencies.list')
-    deps = Open.read(fdep).split("\n") if File.exists?(fdep)
-
-    if done || error || aborted || running || queued || jobid || search
-      select = false
-      select = true if done && exit_status == 0
-      select = true if error && exit_status && exit_status != 0 
-      select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
-      select = true if queued && deps && (running_jobs & deps).any?
-      select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
-      select = true if jobid && jobid.split(",").include?(id)
-      select = true if search && cmd.match(/#{search}/)
-      next unless select 
-    end
-
-
-    puts Log.color :blue, dir
-    puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
-    puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
-    puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
-    puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
-    puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
-    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
-    puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
-    puts Log.color(:magenta, "Nodes: ") << nodes * ", "
-    puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
-
-    if tail && File.exists?(File.join(dir, 'std.err'))
-        if exit_status && exit_status != 0
-            puts Log.color(:magenta, "First error or exception found: ")
-            puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
-        elsif exit_status
-            puts Log.color(:magenta, "Completed jobs: ")
-            puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
-        else
-            puts Log.color(:magenta, "Log tail: ")
-            puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
-        end
-    end
-
-    count += 1
+ dir = File.dirname(fcmd)
+
+ if m = Open.read(fcmd).match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+  exe = m[1]
+ else
+  exe = nil
+ end
+
+ if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+
+ if File.exists?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'job.status'))
+  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exists?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+
+ if done || error || aborted || running || queued || jobid || search
+   select = false
+   select = true if done && exit_status == 0
+   select = true if error && exit_status && exit_status != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   select = true if queued && deps && (running_jobs & deps).any?
+   select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
+   select = true if jobid && jobid.split(",").include?(id)
+   select = true if search && cmd.match(/#{search}/)
+   next unless select 
+ end
+
+
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:sbatch_parameters]
+   puts Log.color(:magenta, "SBATCH parameters: ")
+   puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ end
+
+ if tail && File.exists?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
 
 end
 

data/share/rbbt_commands/slurm/orchestrate

@@ -25,7 +25,7 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -W--workflows* Additional workflows
 -tm--time* Time
--R--orchestration_rules* Orchestration rules
+-OR--orchestration_rules* Orchestration rules
 -rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 
@@ -43,5 +43,5 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/workflow/task

@@ -20,7 +20,7 @@ def usage(workflow = nil, task = nil, exception=nil, abridge = false)
   puts
   if workflow.nil?
     puts "No workflow specified. Use `rbbt workflow list` to list available workflows."
-    exit -1 
+    exit! -1 
   end
 
   if task.nil?
@@ -206,7 +206,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 EOF
 
 workflow = ARGV.shift
-usage and exit -1 if workflow.nil?
+usage and exit! -1 if workflow.nil?
 
 task     = ARGV.shift
 
@@ -232,7 +232,8 @@ else
   remote_workflows = {}
 end
 
-Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+#Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+Workflow.workdir.search_paths.merge!({:workdir => File.expand_path(options.delete(:workdir_all)), :default => :workdir }) if options[:workdir_all]
 
 workflow = Workflow.require_workflow workflow
 
@@ -486,7 +487,7 @@ rescue ParameterException
   puts
   report_options saved_job_options
   puts
-  exit -1
+  exit! -1
 end
 
 if options.delete(:list_job_files)
@@ -538,7 +539,7 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     rescue Exception
       Log.exception $!
@@ -547,9 +548,11 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     end
+  elsif detach
+    exit! 0
   else
     res.join
     out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.29.2
+  version: 5.29.4
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-18 00:00:00.000000000 Z
+date: 2021-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -378,6 +378,7 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
+- share/rbbt_commands/slurm/clean
 - share/rbbt_commands/slurm/list
 - share/rbbt_commands/slurm/orchestrate
 - share/rbbt_commands/slurm/task