rbbt-util 5.30.10 → 5.31.1

data/lib/rbbt/workflow/definition.rb

@@ -79,9 +79,9 @@ module Workflow
       dep = dependencies.last.join
       raise dep.get_exception if dep.error?
       set_info :result_type, dep.info[:result_type]
-      forget = config :forget_dep_tasks, "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
+      forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
-        remove = config :remove_dep_tasks, "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
+        remove = config :remove_dep_tasks, "remove_dep_tasks", "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
 
         self.archive_deps
         self.copy_files_dir

data/lib/rbbt/workflow/step/accessor.rb

@@ -99,6 +99,7 @@ class Step
         if String === value && File.exists?(value)
           Open.ln_s(value, path)
         else
+          value = "#{value}" if Path === value
           Open.write(path + '.yaml', value.to_yaml)
         end
       when Array === value
@@ -122,15 +123,18 @@ class Step
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
     if workflow
-      task_info = workflow.task_info(task_name)
-      input_types = task_info[:input_types]
-      task_inputs = task_info[:inputs]
-      input_defaults = task_info[:input_defaults]
+      task_info = IndiferentHash.setup(workflow.task_info(task_name))
+      input_types = IndiferentHash.setup(task_info[:input_types])
+      task_inputs = IndiferentHash.setup(task_info[:inputs])
+      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
     else
-      task_info = input_types = task_inputs = input_defaults = {}
+      task_info = IndiferentHash.setup({})
+      input_types = IndiferentHash.setup({})
+      task_inputs = IndiferentHash.setup({})
+      input_defaults = IndiferentHash.setup({})
     end
 
-    inputs = {}
+    inputs = IndiferentHash.setup({})
     real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym

data/lib/rbbt/workflow/step/run.rb

@@ -623,6 +623,15 @@ class Step
         Log.warn "Exception removing result of aborted job: #{$!.message}"
       end
     end
+
+    if Open.exists?(tmp_path) && status != :done
+      Log.warn "Aborted job had finished. Removing tmp result -- #{ tmp_path }"
+      begin
+        Open.rm tmp_path
+      rescue Exception
+        Log.warn "Exception removing tmp result of aborted job: #{$!.message}"
+      end
+    end
   end
 
   def _abort

data/lib/rbbt/workflow/usage.rb

@@ -6,19 +6,6 @@ module Task
     puts "\n" << Misc.format_paragraph(description.strip)  << "\n" if description and not description.empty?
     puts
 
-    case
-    when (input_types.values & [:array]).any?
-      puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
-      also works in addition to the others. You may use the '--array_separator' option
-      the change this default. Whenever a file is specified it may also accept STDIN using
-      the '-' character."))
-      puts
-
-    when (input_types.values & [:text, :tsv]).any?
-      puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
-      puts
-    end
-
     selects = []
     if inputs.any?
       inputs.zip(input_types.values_at(*inputs)).select{|i,t| t.to_sym == :select && input_options[i] && input_options[i][:select_options] }.each{|i,t| selects << [i, input_options[i][:select_options]]  }
@@ -50,6 +37,19 @@ module Task
       puts
     end
 
+    case
+    when (input_types.values & [:array]).any?
+      puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
+      also works in addition to the others. You may use the '--array_separator' option
+      the change this default. Whenever a file is specified it may also accept STDIN using
+      the '-' character."))
+      puts
+
+    when (input_types.values & [:text, :tsv]).any?
+      puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
+      puts
+    end
+
     puts Log.color(:magenta, "Returns: ") << Log.color(:blue, result_type.to_s) << "\n"
     puts
 

data/lib/rbbt/workflow/util/provenance.rb

@@ -1,23 +1,28 @@
 class Step
+
+  def self.status_color(status)
+    case status.to_sym
+    when :error, :aborted, :missing, :dead, :unsync
+      :red
+    when :streaming, :started
+      :cyan
+    when :done, :noinfo
+      :green
+    when :dependencies, :waiting, :setup
+      :yellow
+    when :notfound, :cleaned
+      :blue
+    else
+      if status.to_s.index ">"
+        :cyan
+      else
+        :cyan
+      end
+    end
+  end
+
   def self.prov_status_msg(status)
-    color = case status.to_sym
-            when :error, :aborted, :missing, :dead, :unsync
-              :red
-            when :streaming, :started
-              :cyan
-            when :done, :noinfo
-              :green
-            when :dependencies, :waiting, :setup
-              :yellow
-            when :notfound, :cleaned
-              :blue
-            else
-              if status.to_s.index ">"
-                :cyan
-              else
-                :cyan
-              end
-            end
+    color = status_color(status)
     Log.color(color, status.to_s)
   end
 
@@ -25,7 +30,7 @@ class Step
     parts = path.sub(/\{.*/,'').split "/"
 
     parts.pop
-    
+
     task = Log.color(:yellow, parts.pop)
     workflow = Log.color(:magenta, parts.pop)
     #if status.to_s == 'noinfo' && parts.last != 'jobs'
@@ -89,7 +94,7 @@ class Step
         str << prov_report(dep, offset + 1, task, seen, expand_repeats)
       else
         if expand_repeats
-          str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
         else
           info = dep.info  || {}
           status = info[:status] || :missing
@@ -98,7 +103,7 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(status == :notfound ? :blue : :green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
         end
       end
     end if step.dependencies

data/share/config.ru

@@ -43,6 +43,9 @@ app_eval app, Rbbt.etc['app.d/remote_workflow_tasks.rb'].find_all
 #{{{ BASE
 app_eval app, Rbbt.etc['app.d/base.rb'].find
 
+#{{{ SINATRA
+load_file Rbbt.lib['sinatra.rb'].find_all
+
 #{{{ RESOURCES
 load_file Rbbt.etc['app.d/resources.rb'].find
 
@@ -70,9 +73,6 @@ load_file Rbbt.etc['app.d/preload.rb'].find_all
 #{{{ PRELOAD
 load_file Rbbt.etc['app.d/semaphores.rb'].find_all
 
-#{{{ SINATRA
-load_file Rbbt.lib['sinatra.rb'].find_all
-
 Entity.entity_list_cache     = Rbbt.var.sinatra.app[app_name].find.entity_lists
 Entity.entity_map_cache      = Rbbt.var.sinatra.app[app_name].find.entity_maps
 Entity.entity_property_cache = Rbbt.var.sinatra.app[app_name].find.entity_properties

data/share/rbbt_commands/{slurm → hpc}/clean

@@ -2,6 +2,7 @@
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
 
 #$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -9,16 +10,17 @@ options = SOPT.setup <<EOF
 
 Clean error or aborted jobs
 
-$ rbbt mnl [options] 
+$ rbbt slurm clean [options] 
 
 -h--help Print this help
 -d--done Done jobs only
 -e--error Error jobs only
 -a--aborted SLURM aboted jobs
+-q--queued Queued jobs only
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
--SBP--sbatch_parameters show sbatch parameters
+-BP--batch_parameters show batch parameters
 -dr--dry_run Do not erase anything
 EOF
 
@@ -31,14 +33,47 @@ if options[:help]
  exit 0
 end
 
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
 Log.severity = 4
-done, error, aborted, jobid, search, tail, sbatch_parameters, dry_run = options.values_at :done, :error, :aborted, :job, :search, :tail, :sbatch_parameters, :dry_run
+done, error, aborted, queued, jobid, search, tail, batch_parameters, dry_run = options.values_at :done, :error, :aborted, :queued, :job, :search, :tail, :batch_parameters, :dry_run
 
-workdir = File.expand_path('~/rbbt-slurm')
+workdir = File.expand_path('~/rbbt-batch')
 Path.setup(workdir)
 
 running_jobs = begin
-                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt = HPC::BATCH_MODULE.job_status
                 squeue_txt.split("\n").collect{|l| l.to_i.to_s}
                rescue
                 Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
@@ -58,27 +93,35 @@ else
 end
 
 count = 0
-workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  dir = File.dirname(fcmd)
+ command_txt = Open.read(fcmd)
 
- if m = Open.read(fcmd).match(/#CMD: (.*)/)
+ if m = command_txt.match(/#CMD: (.*)/)
   cmd = m[1]
  else
   cmd = nil
  end
 
- if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+ if m = command_txt.match(/# Run command\n(.*?)\n/im)
   exe = m[1]
  else
   exe = nil
  end
 
- if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+ if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
   container_home = m[1]
  else
   container_home = nil
  end
 
+ if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
+  job_batch_system = m[1].downcase
+ else
+  job_batch_system = nil
+ end
+
+ different_system = job_batch_system != batch_system
 
  if File.exists?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
@@ -93,7 +136,16 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  if File.exists?(fstatus = File.join(dir, 'job.status'))
-  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+   fstatus_txt = Open.read(fstatus) 
+   begin
+     if job_batch_system == "lsf"
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
+     else
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+     end
+   rescue
+     nodes = []
+   end
  elsif job_nodes[id]
   nodes = job_nodes[id]
  else
@@ -113,20 +165,36 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
 
  aborted = error = true if aborted.nil? && error.nil?
- if done || error || aborted || running || queued || jobid || search
+ #if done || error || aborted || running || queued || jobid || search
+ #  select = false
+ #  select = true if done && exit_status && exit_status.to_i == 0 
+ #  select = true if error && exit_status && exit_status.to_i != 0 
+ #  select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+ #  select = select && jobid.split(",").include?(id) if jobid 
+ #  select = select && cmd.match(/#{search}/) if search
+ #  next unless select 
+ #end
+
+ if done || error || aborted || queued || jobid 
    select = false
-   select = true if done && exit_status && exit_status.to_i == 0 
-   select = true if error && exit_status && exit_status.to_i != 0 
+   select = true if done && exit_status == 0
+   select = true if error && exit_status && exit_status != 0 
    select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
-   select = select && jobid.split(",").include?(id) if jobid 
+   is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
+   select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
+   select = true if jobid && jobid.split(",").include?(id)
    select = select && cmd.match(/#{search}/) if search
    next unless select 
+ elsif search
+   select = false
+   select = true if search && cmd.match(/#{search}/)
+   next unless select 
  end
 
 
  puts Log.color(:yellow, "**ERASING**")
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
  puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
  puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
@@ -137,9 +205,14 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Nodes: ") << nodes * ", "
  puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
 
- if options[:sbatch_parameters]
-   puts Log.color(:magenta, "SBATCH parameters: ")
-   puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ if options[:batch_parameters]
+   puts Log.color(:magenta, "BATCH parameters: ")
+   case job_batch_system 
+   when 'slurm'
+     puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   when 'lsf'
+     puts Log.color :blue, CMD.cmd('grep "^#BSUB" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   end
  end
 
  if tail && File.exists?(File.join(dir, 'std.err'))

data/share/rbbt_commands/{slurm → hpc}/list

@@ -2,6 +2,7 @@
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
 
 #$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -9,7 +10,7 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt mnl [options] 
+$ rbbt slurm list [options] 
 
 -h--help Print this help
 -d--done Done jobs only
@@ -20,9 +21,11 @@ $ rbbt mnl [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
+-p--progress Report progress of job and the dependencies
+-BP--batch_parameters show batch parameters
+-BPP--batch_procpath show Procpath performance summary
 -sacct--sacct_peformance show sacct performance summary
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]
@@ -34,14 +37,48 @@ if options[:help]
  exit 0
 end
 
-Log.severity = 4
-done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+batch_system = HPC::BATCH_MODULE.to_s.split("::").last.downcase
 
-workdir = File.expand_path('~/rbbt-slurm')
+done, error, running, queued, aborted, jobid, search, tail, progress = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail, :progress
+
+workdir = File.expand_path('~/rbbt-batch')
 Path.setup(workdir)
 
 running_jobs = begin
-                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt = HPC::BATCH_MODULE.job_status
                 squeue_txt.split("\n").collect{|l| l.to_i.to_s}
                rescue
                 Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
@@ -61,35 +98,48 @@ else
 end
 
 count = 0
-workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  dir = File.dirname(fcmd)
+ command_txt = Open.read(fcmd)
 
- if m = Open.read(fcmd).match(/#CMD: (.*)/)
+ if m = command_txt.match(/#CMD: (.*)/)
   cmd = m[1]
  else
   cmd = nil
  end
 
- if m = Open.read(fcmd).match(/#MANIFEST: (.*)/)
+ if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
+  job_batch_system = m[1].downcase
+ else
+  job_batch_system = nil
+ end
+
+ different_system = job_batch_system != batch_system
+
+ if m = command_txt.match(/#MANIFEST: (.*)/)
   manifest = m[1]
  else
   manifest = nil
  end
 
+ if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+ else
+  step_path = nil
+ end
 
- if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
-  exe = m[1].sub('step_path=$(','')
+ if m = command_txt.match(/#EXEC_CMD: (.*)/)
+  exe = m[1]
  else
   exe = nil
  end
 
- if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+ if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
   container_home = m[1]
  else
   container_home = nil
  end
 
-
  if File.exists?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
  else
@@ -103,11 +153,20 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  if File.exists?(fstatus = File.join(dir, 'job.status'))
-  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+   fstatus_txt = Open.read(fstatus) 
+   begin
+     if job_batch_system == "lsf"
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
+     else
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+     end
+   rescue
+     nodes = []
+   end
  elsif job_nodes[id]
-  nodes = job_nodes[id].reject{|n| n.include? "("}
+   nodes = job_nodes[id].reject{|n| n.include? "("}
  else
-  nodes = []
+   nodes = []
  end
 
  if File.exists?(File.join(dir, 'exit.status'))
@@ -136,7 +195,7 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
    select = true if done && exit_status == 0
    select = true if error && exit_status && exit_status != 0 
    select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
-   is_running =  exit_status.nil? && running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)
+   is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
@@ -150,29 +209,39 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
  puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
+ puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
  puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
  puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
  puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
- puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ if different_system
+   puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
+ else
+   puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ end
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", "
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
  puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
 
- if options[:sbatch_parameters]
-   puts Log.color(:magenta, "SBATCH parameters: ")
-   text = CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ if options[:batch_parameters]
+   puts Log.color(:magenta, "BATCH parameters: ")
+   case job_batch_system 
+   when 'slurm'
+     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+   when 'lsf'
+     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+   end
    lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
    puts Log.color :yellow, lines * "\n"
  end
 
  fprocpath = File.join(dir, 'procpath.sqlite3')
- if options[:procpath_performance]  && Open.exists?(fprocpath)
+ if options[:batch_procpath]  && Open.exists?(fprocpath)
    puts Log.color(:magenta, "Procpath summary: ")
    require 'rbbt/tsv/csv'
    meta = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from meta;' "))
@@ -214,13 +283,15 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
  if options[:sacct_peformance]
    begin
-    tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
-    values = tsv[tsv.keys.first]
-    if values.compact.any?
-     puts Log.color(:magenta, "SACCT performance: ")
-     puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ")  + v.to_s } * "\n"
-    end
+     raise "sacct not supported for LSF" unless batch_system == 'slurm' 
+     tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
+     values = tsv[tsv.keys.first]
+     if values.compact.any?
+       puts Log.color(:magenta, "SACCT performance: ")
+       puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ")  + v.to_s } * "\n"
+     end
    rescue
+     Log.warn $!.message
    end
  end
 
@@ -234,7 +305,24 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | tail -n #{tail.to_i} ").read
+   end
+ end
+
+ if options[:progress] 
+   step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+   if step_line
+     require 'rbbt/workflow'
+     step_path = step_line.split(": ").last.strip
+     step = Step.new step_path
+     step.load_dependencies_from_info
+     (step.rec_dependencies + [step]).reverse.each do |j|
+       next if j.done? 
+       next unless j.file(:progress).exists?
+       bar = Log::ProgressBar.new 
+       bar.load(j.file(:progress).yaml)
+       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+     end
    end
  end