rbbt-util 5.33.3 → 5.33.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ffb05883cf1e7eda628395b50967322b42240ace100fb73a2f4c4a1ed5e8d27
-  data.tar.gz: 8b25c002f68b88dc0ba1f09c041258ee0ca15d4fc6f8dc3df5de37ea2473d059
+  metadata.gz: 7ef9895840235ea220e85895364588fca0c56b1ed8673483474465bfbf94a32b
+  data.tar.gz: 68ba6769881c9118256efd14c8d05ed1ca6f8ab56733800c45c25631016647d0
 SHA512:
-  metadata.gz: 7b74e527336830eecb1cad52bb13cbcb335c734a18e9261d87cfba6da93422cf94a826e97d43c6ef1d37d122623408230d80a037a9a91000fa95a6cf9c818cf6
-  data.tar.gz: 8b8de5c12aac8dfae5f68018cb5aae4023b78993fa78aa7cf93cacc16eea4c2b959f75190ead6862502ecc75ae3ccd9cc462b1db26930943447a758f1f359cd9
+  metadata.gz: 53c4c81fcfafd8e001f33886323829e230a93731ba23379e1f40358f3bcfae72931d3978d2db3a490b48f22c618462947d5f63a3cc7d135564588d4f6b48aae8
+  data.tar.gz: c53364bbc000b098d4b80afaab6fa8fb16802f9106bfcb8a0aa847b6b8d970e373f102a0fec6021e817a6ebec58d7dc7586c315c9f3aee67481d975d02558abc

data/lib/rbbt/hpc/batch.rb

@@ -538,10 +538,13 @@ env > #{batch_options[:fenv]}
       workflow = job.workflow
       task_name = job.task_name
 
+      workflows_to_load = job.rec_dependencies.select{|d| Step === d}.collect{|d| d.workflow }.compact.collect(&:to_s) - [workflow.to_s]
+
       TmpFile.with_file(nil, remove_batch_dir, :tmpdir => batch_base_dir, :prefix => "#{system}_rbbt_job-#{workflow.to_s}-#{task_name}-") do |batch_dir|
         Misc.add_defaults options, 
           :batch_dir => batch_dir, 
-          :inputs_dir => File.join(batch_dir, "inputs_dir")
+          :inputs_dir => File.join(batch_dir, "inputs_dir"),
+          :workflows => workflows_to_load * ","
 
         options[:procpath_performance] ||= File.join(batch_dir, "procpath##{procpath.gsub(',', '#')}") if procpath
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -31,7 +31,8 @@ module HPC
         top = batches.select{|b| b[:deps].nil? || (b[:deps] - batch_ids.keys).empty? }.first
         raise "No batch without unmet dependencies" if top.nil?
         batches.delete top
-        job_options = options.merge(top[:rules])
+
+        job_options = HPC::Orchestration.merge_rules(options, top[:rules])
 
         if top[:deps].nil?
           batch_dependencies = [] 
@@ -59,7 +60,9 @@ module HPC
 
         if options[:dry_run]
           puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-          puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          puts Log.color(:magenta, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          puts Log.color(:yellow, "Path: ") + top[:top_level].path
+          puts Log.color(:yellow, "Options: ") + Misc.fingerprint(job_options)
           batch_ids[top] = top[:top_level].task_signature
         else
           id = run_job(top[:top_level], job_options)

data/lib/rbbt/hpc/slurm.rb

@@ -32,6 +32,8 @@ export BATCH_SYSTEM=SLURM
       constraint     = Misc.process_options options, :constraint
       gres           = Misc.process_options options, :gres
 
+      constraint     = [constraint, "highmem"].compact * "&" if highmem
+
       mem            = Misc.process_options options, :mem
       mem_per_cpu    = Misc.process_options options, :mem_per_cpu
 
@@ -50,14 +52,15 @@ export BATCH_SYSTEM=SLURM
                        "cpus-per-task" => task_cpus,
                        "nodes" => nodes,
                        "time" => time,
+                       "constraint" => constraint,
                        "exclusive" => exclusive,
-                       "highmem" => highmem,
                        "licenses" => licenses,
                        "gres" => gres,
                        "mem" => mem,
                        "mem-per-cpu" => mem_per_cpu,
       }
 
+
       header =<<-EOF
 #!/bin/bash
       EOF

data/lib/rbbt/resource.rb

@@ -270,6 +270,7 @@ source "$INSTALL_HELPER_FILE"
                               git = content[:git]
                               src = content[:src]
                               url = content[:url]
+                              jar = content[:jar]
                               extra = content[:extra]
                               commands = content[:commands]
                               if git
@@ -290,6 +291,16 @@ url='#{src}'
 
 install_src "$name" "$url" #{extra}
 
+#{commands}
+                                EOF
+                              elsif jar
+                                <<-EOF
+
+name='#{name}'
+url='#{jar}'
+
+install_jar "$name" "$url" #{extra}
+
 #{commands}
                                 EOF
                               else

data/lib/rbbt/workflow/definition.rb

@@ -93,22 +93,22 @@ module Workflow
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
 
-        if dep.overriden
+        if dep.overriden || ! Workflow.job_path?(dep.path)
           Open.link dep.path, self.tmp_path
         else
           Open.ln_h dep.path, self.tmp_path
-        end
 
-        case remove.to_s
-        when 'true'
-          dep.clean
-        when 'recursive'
-          (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
-            next if d.overriden
-            d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
-          end
-          dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
-        end unless dep.overriden
+          case remove.to_s
+          when 'true'
+            dep.clean
+          when 'recursive'
+            (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+              next if d.overriden
+              d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
+            end
+            dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
+          end 
+        end
       else
         if Open.exists?(dep.files_dir)
           Open.rm_rf self.files_dir 

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -5,10 +5,10 @@ module Workflow
 
     def self.job_workload(job)
       workload = {job => []}
-      return workload if job.done?
+      return workload if job.done? && ! job.dirty?
 
       job.dependencies.each do |dep|
-        next if dep.done?
+        next if dep.done? && ! job.dirty?
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep
@@ -16,7 +16,7 @@ module Workflow
       end
 
       job.input_dependencies.each do |dep|
-        next if dep.done?
+        next if dep.done? && ! job.dirty?
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep

data/share/Rlib/plot.R

@@ -38,3 +38,4 @@ geom_entity <- function (real.geom = NULL, mapping = NULL, data = NULL, stat = "
     rg
 }
 
+rbbt.ggplot2.rotate_x_labels <- function(){ theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) }

data/share/rbbt_commands/hpc/list

@@ -21,6 +21,7 @@ $ rbbt slurm list [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-l--long Show more entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -40,6 +41,8 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
+long = options.delete :long
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -183,24 +186,24 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
- puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
- puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
+ puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
+ puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) if long
  puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
- puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
- puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
- puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing") if long
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing") if long
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home && long
  if different_system
    puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
  else
    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
- end
+ end 
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
- puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -281,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/hpc/tail

@@ -10,7 +10,7 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt slurm tail <directory> [options] 
+$ rbbt slurm tail <directory|jobid> [options] 
 
 -h--help Print this help
 EOF
@@ -35,8 +35,28 @@ directory = ARGV.shift
 
 raise ParameterException if directory.nil?
 
-directory = File.dirname(directory) unless File.directory?(directory)
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/rbbt-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
 
 require 'rbbt/hpc/slurm'
 
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/lsf/list

@@ -21,6 +21,7 @@ $ rbbt slurm list [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-l--long Show more entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -40,6 +41,8 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
+long = options.delete :long
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -183,24 +186,24 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
- puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
- puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
+ puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
+ puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) if long
  puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
- puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
- puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
- puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing") if long
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing") if long
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home && long
  if different_system
    puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
  else
    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
- end
+ end 
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
- puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -281,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/lsf/tail

@@ -10,7 +10,7 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt slurm tail <directory> [options] 
+$ rbbt slurm tail <directory|jobid> [options] 
 
 -h--help Print this help
 EOF
@@ -35,8 +35,28 @@ directory = ARGV.shift
 
 raise ParameterException if directory.nil?
 
-directory = File.dirname(directory) unless File.directory?(directory)
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/rbbt-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
 
 require 'rbbt/hpc/slurm'
 
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/slurm/list

@@ -21,6 +21,7 @@ $ rbbt slurm list [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-l--long Show more entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -40,6 +41,8 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
+long = options.delete :long
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -183,24 +186,24 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
- puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
- puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
+ puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
+ puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) if long
  puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
- puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
- puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
- puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing") if long
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing") if long
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home && long
  if different_system
    puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
  else
    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
- end
+ end 
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
- puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -281,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/slurm/tail

@@ -10,7 +10,7 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt slurm tail <directory> [options] 
+$ rbbt slurm tail <directory|jobid> [options] 
 
 -h--help Print this help
 EOF
@@ -35,8 +35,28 @@ directory = ARGV.shift
 
 raise ParameterException if directory.nil?
 
-directory = File.dirname(directory) unless File.directory?(directory)
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/rbbt-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
 
 require 'rbbt/hpc/slurm'
 
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/workflow/prov

@@ -37,7 +37,7 @@ file = ARGV.shift
 $seen = []
 def get_step(file)
   file = File.expand_path(file)
-  file = file.sub(/\.(info|files)/,'')
+  file = file.sub(/\.(info|files)$/,'')
   $seen << file
   Workflow.load_step file
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.3
+  version: 5.33.6
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-28 00:00:00.000000000 Z
+date: 2022-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake