rbbt-util 5.33.2 → 5.33.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 331adb8d6a45a7cc3ee18b17785ffe03d28585274e9b2d1cab5cd502cdc90821
-  data.tar.gz: 8580df0b38d805739fe3ad80e688c92a5c31305893ef1cf6cfccd88c0aaccc7a
+  metadata.gz: bea55b2e4dd5972cf0e9646e84db4ce6ffd0556bb54796927640dbe3a3ba6611
+  data.tar.gz: 281d52e1685badaba8427e5ffd2226e1a1ab61d66df30cc7cd88acb79ef5aa18
 SHA512:
-  metadata.gz: 6381f886a7282c03c8e8303ae148fe4a4fc3b82dad60d1d039953d83075aa85ccc19bb75f24811e2158a6715b9ed471288b2b23110c981a61ad167bc6593aac0
-  data.tar.gz: 1476c3fdf9fe56928a9dd85fc2384735b71a0a40a45359197fc203608f2933aa6a6f5ec94c1bd5ae4b5782b2cd2bbe752f2d06973f9989307d81e3a1f261eba2
+  metadata.gz: 459b75076980833dc1be3bd60d9fc98707270c626ab837e76a6c09180aaa2bd3328b4dea8776cf09c8fca0358e38258a30a90d17cd615450a38398c53bacfef0
+  data.tar.gz: 1bb336e1a592b64fbb3e87453042e3f6f220b76c92f5e9ff7550aa25401fd3e36a823f31bddb0e8be6878fa7646b28a010ac320d6fd8e00066bd1ebf2e9f774f

data/lib/rbbt/hpc/batch.rb

@@ -538,10 +538,13 @@ env > #{batch_options[:fenv]}
       workflow = job.workflow
       task_name = job.task_name
 
+      workflows_to_load = job.rec_dependencies.select{|d| Step === d}.collect{|d| d.workflow }.compact.collect(&:to_s) - [workflow.to_s]
+
       TmpFile.with_file(nil, remove_batch_dir, :tmpdir => batch_base_dir, :prefix => "#{system}_rbbt_job-#{workflow.to_s}-#{task_name}-") do |batch_dir|
         Misc.add_defaults options, 
           :batch_dir => batch_dir, 
-          :inputs_dir => File.join(batch_dir, "inputs_dir")
+          :inputs_dir => File.join(batch_dir, "inputs_dir"),
+          :workflows => workflows_to_load * ","
 
         options[:procpath_performance] ||= File.join(batch_dir, "procpath##{procpath.gsub(',', '#')}") if procpath
 

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -11,7 +11,7 @@ module HPC
         iii :END_BATCHES
       else
         n = batch.dup
-        n[:deps] = n[:deps].collect{|b| b[:top_level] }
+        n[:deps] = n[:deps].collect{|b| b[:top_level] } if n[:deps]
         iif n
       end
     end
@@ -79,14 +79,14 @@ module HPC
 
       batches.each do |batch|
         jobs = batch[:jobs]
-        all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
+        all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq - jobs
 
         minimum = all_deps
         all_deps.each do |dep|
           minimum -= job_dependencies(dep)
         end
 
-        all_deps = minimum
+        all_deps = minimum 
         deps = all_deps.collect do |d|
           (batches - [batch]).select{|batch| batch[:jobs].collect(&:path).include? d.path }
         end.flatten.uniq

data/lib/rbbt/hpc/orchestrate.rb

@@ -31,7 +31,8 @@ module HPC
         top = batches.select{|b| b[:deps].nil? || (b[:deps] - batch_ids.keys).empty? }.first
         raise "No batch without unmet dependencies" if top.nil?
         batches.delete top
-        job_options = options.merge(top[:rules])
+
+        job_options = HPC::Orchestration.merge_rules(options, top[:rules])
 
         if top[:deps].nil?
           batch_dependencies = [] 

data/lib/rbbt/hpc/slurm.rb

@@ -32,6 +32,8 @@ export BATCH_SYSTEM=SLURM
       constraint     = Misc.process_options options, :constraint
       gres           = Misc.process_options options, :gres
 
+      constraint     = [constraint, "highmem"].compact * "&" if highmem
+
       mem            = Misc.process_options options, :mem
       mem_per_cpu    = Misc.process_options options, :mem_per_cpu
 
@@ -50,14 +52,15 @@ export BATCH_SYSTEM=SLURM
                        "cpus-per-task" => task_cpus,
                        "nodes" => nodes,
                        "time" => time,
+                       "constraint" => constraint,
                        "exclusive" => exclusive,
-                       "highmem" => highmem,
                        "licenses" => licenses,
                        "gres" => gres,
                        "mem" => mem,
                        "mem-per-cpu" => mem_per_cpu,
       }
 
+
       header =<<-EOF
 #!/bin/bash
       EOF

data/lib/rbbt/resource.rb

@@ -270,6 +270,7 @@ source "$INSTALL_HELPER_FILE"
                               git = content[:git]
                               src = content[:src]
                               url = content[:url]
+                              jar = content[:jar]
                               extra = content[:extra]
                               commands = content[:commands]
                               if git
@@ -290,6 +291,16 @@ url='#{src}'
 
 install_src "$name" "$url" #{extra}
 
+#{commands}
+                                EOF
+                              elsif jar
+                                <<-EOF
+
+name='#{name}'
+url='#{jar}'
+
+install_jar "$name" "$url" #{extra}
+
 #{commands}
                                 EOF
                               else

data/lib/rbbt/util/cmd.rb

@@ -75,7 +75,7 @@ module CMD
 
     string = ""
     options.each do |option, value|
-      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=]+$/i
+      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
       #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
       value = value.gsub("'","\\'") if value.to_s.include? "'"
 

data/lib/rbbt/workflow/definition.rb

@@ -92,18 +92,23 @@ module Workflow
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
-        Open.ln_h dep.path, self.tmp_path
-
-        case remove.to_s
-        when 'true'
-          dep.clean
-        when 'recursive'
-          (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
-            next if d.overriden
-            d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
-          end
-          dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
-        end unless dep.overriden
+
+        if dep.overriden || ! Workflow.job_path?(dep.path)
+          Open.link dep.path, self.tmp_path
+        else
+          Open.ln_h dep.path, self.tmp_path
+
+          case remove.to_s
+          when 'true'
+            dep.clean
+          when 'recursive'
+            (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+              next if d.overriden
+              d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
+            end
+            dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
+          end 
+        end
       else
         if Open.exists?(dep.files_dir)
           Open.rm_rf self.files_dir 

data/lib/rbbt/workflow/integration/nextflow.rb

@@ -4,7 +4,7 @@ module Workflow
   end
 
   def self.nextflow_includes(file)
-    Open.read(file).scan(/^include\s*{\s*(.*?)(?:\s*as.*?)?}\s*from\s+"(.*?)"(?:\s*params.*)?/).collect{|p| p}.uniq
+    Open.read(file).scan(/^include\s*{\s*([^\s]*?)\s+.*?}\s*from\s+["'](.*?)["'](?:\s*params.*)?/).collect{|p| p}.uniq
   end
 
   def self.nextflow_recursive_params(file)
@@ -48,25 +48,43 @@ module Workflow
     task name => result do 
       work = file('work')
       profile = config :profile, :nextflow
+      config_file = config :config, :nextflow
 
-      new_inputs = inputs.zip(inputs.fields).collect do |v,f|
-        if String === v && m = v.match(/^JOB_FILE:(.*)/)
-          file(m[1]) 
-        elsif v.nil?
-          Rbbt::Config.get(['nextflow', f] * "_", 'default', f)
+      nextflow_inputs = {}
+
+      inputs.zip(inputs.fields).collect do |v,f|
+        v = if String === v && m = v.match(/^JOB_FILE:(.*)/)
+              file(m[1]) 
+            elsif v.nil?
+              Rbbt::Config.get(['nextflow', f] * "_", 'default', f)
+            else
+              v
+            end
+
+        if f.to_s.include?("-") 
+          p,_sep, section = f.to_s.partition("-")
+          name = [section, p] * "."
         else
-          v
+          name = f
         end
+          
+        nextflow_inputs[name] = v
       end
       
-      inputs.replace new_inputs
-
       Misc.in_dir file('stage') do
-        if profile
-          cmd("nextflow run -work-dir #{work} -ansi-log false  -profile #{profile} #{file}", inputs.to_hash.merge('add_option_dashes' => true))
-        else
-          cmd("nextflow run -work-dir #{work} -ansi-log false #{file}", inputs.to_hash.merge('add_option_dashes' => true))
-        end
+
+        cmd = "nextflow "
+
+        cmd += " -C #{config_file}" if config_file
+
+        cmd += " run"
+
+        cmd += " -work-dir #{work} -ansi-log false"
+
+        cmd += " -profile #{profile}" if profile
+
+
+        cmd("#{cmd} #{file}", nextflow_inputs.merge('add_option_dashes' => true))
       end
 
       output_file = file(output).glob.first if output

data/share/rbbt_commands/hpc/list

@@ -21,6 +21,7 @@ $ rbbt slurm list [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-l--long Show more entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -40,6 +41,8 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
+long = options.delete :long
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -183,24 +186,24 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
- puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
- puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
+ puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
+ puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) if long
  puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
- puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
- puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
- puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing") if long
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing") if long
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home && long
  if different_system
    puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
  else
    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
- end
+ end 
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
- puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -281,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/hpc/tail

@@ -10,7 +10,7 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt slurm tail <directory> [options] 
+$ rbbt slurm tail <directory|jobid> [options] 
 
 -h--help Print this help
 EOF
@@ -35,8 +35,28 @@ directory = ARGV.shift
 
 raise ParameterException if directory.nil?
 
-directory = File.dirname(directory) unless File.directory?(directory)
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/rbbt-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
 
 require 'rbbt/hpc/slurm'
 
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/lsf/list

@@ -21,6 +21,7 @@ $ rbbt slurm list [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-l--long Show more entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -40,6 +41,8 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
+long = options.delete :long
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -183,24 +186,24 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
- puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
- puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
+ puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
+ puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) if long
  puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
- puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
- puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
- puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing") if long
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing") if long
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home && long
  if different_system
    puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
  else
    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
- end
+ end 
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
- puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -281,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/lsf/tail

@@ -10,7 +10,7 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt slurm tail <directory> [options] 
+$ rbbt slurm tail <directory|jobid> [options] 
 
 -h--help Print this help
 EOF
@@ -35,8 +35,28 @@ directory = ARGV.shift
 
 raise ParameterException if directory.nil?
 
-directory = File.dirname(directory) unless File.directory?(directory)
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/rbbt-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
 
 require 'rbbt/hpc/slurm'
 
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/slurm/list

@@ -21,6 +21,7 @@ $ rbbt slurm list [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-l--long Show more entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -40,6 +41,8 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
+long = options.delete :long
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -183,24 +186,24 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
- puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
- puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
+ puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
+ puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) if long
  puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
- puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
- puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
- puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing") if long
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing") if long
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home && long
  if different_system
    puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
  else
    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
- end
+ end 
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
- puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -281,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/slurm/tail

@@ -10,7 +10,7 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt slurm tail <directory> [options] 
+$ rbbt slurm tail <directory|jobid> [options] 
 
 -h--help Print this help
 EOF
@@ -35,8 +35,28 @@ directory = ARGV.shift
 
 raise ParameterException if directory.nil?
 
-directory = File.dirname(directory) unless File.directory?(directory)
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/rbbt-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
 
 require 'rbbt/hpc/slurm'
 
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
 HPC::BATCH_MODULE.follow_job directory, true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.2
+  version: 5.33.5
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-23 00:00:00.000000000 Z
+date: 2022-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake