rbbt-util 5.40.0 → 5.40.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '088ac190b36a176e420d4bb1d7b8d28d777133568e45530219f6a0212a1a1144'
-  data.tar.gz: 47bdc31d247e5fd5d944f493690e8181bcc236cee74c61fefd5ce25711edf7da
+  metadata.gz: 0c086b1d2a4bfd64cc80729a4962b2ba04268b5745ae6c79b5d954fc5b15a7a9
+  data.tar.gz: 8d0ee0a0942be2cf07259baed09d2c2feb98a67b314adc70bd0ac1e03ecba4f9
 SHA512:
-  metadata.gz: 9afce115c221695ee896c1e1f884be9a1ba0f0ae98789a485a906e438252afe8292c5422f3eef57d77089bf55cc0b425988ab9b479b1f422eaa0abb7005cfdfa
-  data.tar.gz: 4cf4d1e86ce3ab256609aff4c66a3412ea1a9c0c87db08fd5cdad1b98c95147d452cc69d96009c9791cf698ea43903873ed0e4cebbc861bda85097864bed68e8
+  metadata.gz: 8ca4b7b1b59eb4546b9e1484fd397c687003389b406de77461da5aa3c773679a87533050c640c90feacc48f9bdadf1c565bcfe9727e8661b2440938ed3c69dbc
+  data.tar.gz: 613f30647a8d317984e12a5618fe8d03ab94c3d4ca1acc8ee38460c8f67869c13e8dd83f3d1c4332e813a34802af55d6346f87d898863095ff12368c9f715ba6

data/bin/rbbt

@@ -75,7 +75,7 @@ end
 Log.ignore_stderr do
   begin
     require "nokogiri"
-  rescue
+  rescue Exception
   end
 end
 

data/lib/rbbt/hpc/pbs.rb

@@ -43,6 +43,8 @@ cd ${PBS_O_WORKDIR}
       system     = Misc.process_options options, :partition
       filesystems = Misc.process_options options, :filesystems
 
+      filesystems = "home" if filesystems.nil?
+
       filesystems = filesystems * "," if Array === filesystems
 
       # NOT USED 

data/lib/rbbt/util/misc/development.rb

@@ -1,6 +1,6 @@
 module Misc
 
-def self.add_libdir(dir=nil)
+  def self.add_libdir(dir=nil)
     dir ||= File.join(Path.caller_lib_dir(caller.first), 'lib')
     $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include? dir
   end

data/lib/rbbt/util/python/step.rb

@@ -0,0 +1,41 @@
+require 'rbbt/workflow/step'
+
+class Step
+  def python_file(file, options = {})
+    CMD.cmd_log(:python, file, options)
+  end
+
+  def python_block(options = {}, &block)
+    RbbtPython.run options, &block
+  end
+
+  def python(python = nil, options = {}, &block)
+    begin
+      RbbtPython.add_path self.workflow.libdir.python.find
+    rescue
+      Log.warn "Error loading libdir python for workflow '#{Misc.fingerprint self.workflow}'"
+    end
+    case python
+    when Path
+      python_file python.find, options
+    when String
+      if Open.exists?(python)
+        python_file python
+      else
+        TmpFile.with_file do |dir|
+          pkg = "pkg#{rand(100)}"
+          Open.write File.join(dir, "#{pkg}/__init__.py"), code
+
+          RbbtPython.add_path dir
+
+          Misc.in_dir dir do
+            yield pkg
+          end
+        end
+      end
+    else
+      python_block(python, &block)
+    end
+  end
+end
+

data/lib/rbbt/util/python/util.rb

@@ -49,5 +49,5 @@ module RbbtPython
     end
     hash
   end
-
 end
+

data/python/rbbt.py

@@ -29,6 +29,7 @@ def tsv_preamble(line, comment_char="#"):
     entries = re.sub(f"^{comment_char}:", '', line)
     entries = re.sub(f"^{comment_char}:", '', line).split("#")
     for entry in entries:
+        entry = entry.strip()
         key, value = entry.split("=")
         key = re.sub("^:","",key)
         value = re.sub("^:","",value)
@@ -75,7 +76,13 @@ def tsv_pandas(filename, sep="\t", comment_char="#", index_col=0, **kwargs):
         header = tsv_header(filename, sep=sep, comment_char="#")
 
         if ("type" in header and header["type"] == "flat"):
-            return None
+            if ("sep" in header):
+                sep=header["sep"]
+
+            tsv = pandas.read_table(filename, sep=sep, index_col=index_col, header=None, skiprows=[0,1], **kwargs)
+
+            if ("key_field" in header):
+                tsv.index.name = header["key_field"]
         else:
             if ("sep" in header):
                 sep=header["sep"]

data/share/rbbt_commands/pbs/clean

@@ -0,0 +1,212 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean error or aborted jobs
+
+$ rbbt slurm clean [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-BP--batch_parameters show batch parameters
+-dr--dry_run Do not erase anything
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+Log.severity = 4
+done, error, aborted, queued, jobid, search, tail, batch_parameters, dry_run = options.values_at :done, :error, :aborted, :queued, :job, :search, :tail, :batch_parameters, :dry_run
+
+workdir = File.expand_path('~/rbbt-batch')
+Path.setup(workdir)
+
+running_jobs = begin
+                squeue_txt = HPC::BATCH_MODULE.job_status
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
+               end
+
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
+count = 0
+workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
+ dir = File.dirname(fcmd)
+ command_txt = Open.read(fcmd)
+
+ if m = command_txt.match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = command_txt.match(/# Run command\n(.*?)\n/im)
+  exe = m[1]
+ else
+  exe = nil
+ end
+
+ if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+ if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
+  job_batch_system = m[1].downcase
+ else
+  job_batch_system = nil
+ end
+
+ different_system = job_batch_system != batch_system
+
+ if File.exist?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exist?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exist?(fstatus = File.join(dir, 'job.status'))
+   fstatus_txt = Open.read(fstatus) 
+   begin
+     if job_batch_system == "lsf"
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
+     else
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+     end
+   rescue
+     nodes = []
+   end
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exist?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exist?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
+
+ aborted = error = true if ! done && aborted.nil? && error.nil?
+ #if done || error || aborted || running || queued || jobid || search
+ #  select = false
+ #  select = true if done && exit_status && exit_status.to_i == 0 
+ #  select = true if error && exit_status && exit_status.to_i != 0 
+ #  select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+ #  select = select && jobid.split(",").include?(id) if jobid 
+ #  select = select && cmd.match(/#{search}/) if search
+ #  next unless select 
+ #end
+
+ if done || error || aborted || queued || jobid 
+   select = false
+   select = true if done && exit_status == 0
+   select = true if error && exit_status && exit_status != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
+   select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
+   select = true if jobid && jobid.split(",").include?(id)
+   select = select && cmd.match(/#{search}/) if search
+   next unless select 
+ elsif search
+   select = false
+   select = true if search && cmd.match(/#{search}/)
+   next unless select 
+ end
+
+
+ puts Log.color(:yellow, "**ERASING**")
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exist?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:batch_parameters]
+   puts Log.color(:magenta, "BATCH parameters: ")
+   case job_batch_system 
+   when 'slurm'
+     puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   when 'lsf'
+     puts Log.color :blue, CMD.cmd('grep "^#BSUB" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   end
+ end
+
+ if tail && File.exist?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
+
+ Open.rm_rf dir unless dry_run
+end
+
+puts 
+puts "Found #{count} jobs"
+

data/share/rbbt_commands/pbs/list

@@ -0,0 +1,362 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm list [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-l--long Show more entries
+-c--compressed Show compressed information about entries
+-p--progress Report progress of job and the dependencies
+-BP--batch_parameters show batch parameters
+-BPP--batch_procpath show Procpath performance summary
+-sacct--sacct_peformance show sacct performance summary
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+long = options.delete :long
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+batch_system = HPC::BATCH_MODULE.to_s.split("::").last.downcase
+
+done, error, running, queued, aborted, jobid, search, tail, progress = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail, :progress
+
+workdir = File.expand_path('~/rbbt-batch')
+Path.setup(workdir)
+
+running_jobs = begin
+                 squeue_txt = HPC::BATCH_MODULE.job_status
+                 squeue_txt.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                 Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                 squeue_txt = nil
+                 $norunningjobs = true
+                 []
+               end
+
+if squeue_txt
+  job_nodes = {}
+  squeue_txt.split("\n").each do |line|
+    parts = line.strip.split(/\s+/)
+    job_nodes[parts.first] = parts.last.split(",")
+  end
+else
+  job_nodes = nil
+end
+
+count = 0
+workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
+  dir = File.dirname(fcmd)
+  command_txt = Open.read(fcmd)
+
+  if m = command_txt.match(/#CMD: (.*)/)
+    cmd = m[1]
+  else
+    cmd = nil
+  end
+
+  if m = command_txt.match(/^export BATCH_SYSTEM=(.*)/)
+    job_batch_system = m[1].downcase
+  else
+    job_batch_system = nil
+  end
+
+  different_system = job_batch_system != batch_system
+
+  if m = command_txt.match(/#MANIFEST: (.*)/)
+    manifest = m[1]
+  else
+    manifest = nil
+  end
+
+  if m = command_txt.match(/#STEP_PATH: (.*)/)
+    step_path = m[1]
+  else
+    step_path = nil
+  end
+
+  if m = command_txt.match(/#EXEC_CMD: (.*)/)
+    exe = m[1]
+  else
+    exe = nil
+  end
+
+  if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
+    container_home = m[1]
+  else
+    container_home = nil
+  end
+
+  if File.exist?(fid = File.join(dir, 'job.id'))
+    id = Open.read(fid).chomp
+  else
+    id = nil
+  end
+
+  if File.exist?(fstatus = File.join(dir, 'exit.status'))
+    exit_status = Open.read(fstatus).to_i
+  else
+    exit_status = nil
+  end
+
+  if File.exist?(fstatus = File.join(dir, 'job.status'))
+    fstatus_txt = Open.read(fstatus) 
+    begin
+      if job_batch_system == "lsf"
+        nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
+      else
+        nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+      end
+    rescue
+      nodes = []
+    end
+  elsif job_nodes && job_nodes[id]
+    nodes = job_nodes[id].reject{|n| n.include? "("}
+  else
+    nodes = []
+  end
+
+  if File.exist?(File.join(dir, 'exit.status'))
+    now = File.ctime(File.join(dir, 'exit.status'))
+  else
+    now = Time.now
+  end
+
+  if File.exist?(File.join(dir, 'std.out'))
+    cerrt = File.ctime File.join(dir, 'std.err')
+    coutt = File.ctime File.join(dir, 'std.out')
+    outt = File.mtime File.join(dir, 'std.out')
+    errt = File.mtime File.join(dir, 'std.err')
+    time_diff = now - [outt, errt].max
+    time_elapsed = now - [cerrt, coutt].min
+  end
+
+  fdep = File.join(dir, 'dependencies.list')
+  deps = Open.read(fdep).split("\n") if File.exist?(fdep)
+
+  fcadep = File.join(dir, 'canfail_dependencies.list')
+  cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
+
+  is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (deps.nil? || (running_jobs & deps).empty?)) || different_system )
+  if done || error || aborted || running || queued || jobid 
+    select = false
+    select = true if done && exit_status == 0
+    select = true if error && exit_status && exit_status != 0 
+    select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
+    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
+    select = true if jobid && jobid.split(",").include?(id)
+    select = select && step_path.match(/#{search}/) if search
+    next unless select 
+  elsif search
+    select = false
+    select = true if search && cmd.match(/#{search}/)
+    next unless select 
+  end
+
+
+  count += 1
+
+  if options[:compressed] 
+    status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
+    if different_system
+      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
+    else
+      #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
+        (running_jobs.include?(id) || $norunningjobs ? 
+         (is_running ? Log.color(:cyan, id) : Log.color(:yellow, id) ) : 
+         Log.color(:red, id) )
+    end 
+    prog_rep = []
+    if options[:progress] 
+      step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+      if step_line
+        require 'rbbt/workflow'
+        step_path = step_line.split(": ").last.strip
+        step = Step.new step_path
+        step.load_dependencies_from_info
+        has_bar = false
+        [step].reverse.each do |j|
+          next if j.done? 
+          if j.file(:progress).exists?
+            bar = Log::ProgressBar.new 
+            bar.load(j.file(:progress).yaml)
+            rep = bar.report_msg.split("·")[1]
+            rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
+            prog_rep << [rep]
+          end
+        end
+      end
+    end
+    workflow, task, name = step_path.split("/")[-3..-1]
+    job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
+    puts [job_str, status, prog_rep ].flatten * " "
+    next
+  end
+
+  puts Log.color :blue, dir
+  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
+  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
+  puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) if long
+  puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
+  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+  puts Log.color(:magenta, "Exec: ") << (exe || "Missing") if long
+  puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing") if long
+  puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home && long
+  if different_system
+    puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
+  else
+    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+  end 
+  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+  puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
+  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
+  puts Log.color(:magenta, "Output: ") << File.exist?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exist?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
+
+  if options[:batch_parameters]
+    puts Log.color(:magenta, "BATCH parameters: ")
+    case job_batch_system 
+    when 'slurm'
+      text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
+    when 'lsf'
+      text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
+    when 'pbs'
+      text = CMD.cmd('grep "^#PBS" ', :in => Open.read(fcmd)).read.strip
+    else
+      text = ""
+    end
+    lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
+    puts Log.color :yellow, lines * "\n"
+  end
+
+  fprocpath = File.join(dir, 'procpath.sqlite3')
+  if options[:batch_procpath]  && Open.exists?(fprocpath)
+    puts Log.color(:magenta, "Procpath summary: ")
+    require 'rbbt/tsv/csv'
+    meta = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from meta;' "))
+    perf = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from record;' "))
+
+    page_size = meta["page_size"].first.to_f
+    clock_ticks = meta["clock_ticks"].first.to_f
+
+    cpu_average = {}
+    rss_average = {}
+    perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
+      time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
+      time = time.to_f
+
+      cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
+      cpu_average[stat_pid] ||= {}
+      cpu_average[stat_pid][time] ||= []
+      cpu_average[stat_pid][time] << cpu.to_f
+      rss_average[time] ||= []
+      rss_average[time] << rss.to_f * page_size
+    end
+
+    ticks = 0
+    cpu_average.each do |stat_pid, cpu_average_pid|
+      start = cpu_average_pid.keys.sort.first
+      eend = cpu_average_pid.keys.sort.last
+      ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
+    end
+    start = rss_average.keys.sort.first
+    eend = rss_average.keys.sort.last
+    time_elapsed = eend - start
+    ticks = 1 if ticks == 0
+    time_elapsed = 1 if time_elapsed == 0
+    puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
+    puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s 
+    puts Log.color(:yellow, "Time: ") + Misc.format_seconds((eend - start))
+
+  end
+
+  if options[:sacct_peformance]
+    begin
+      raise "sacct not supported for LSF" unless batch_system == 'slurm' 
+      tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
+      values = tsv[tsv.keys.first]
+      if values.compact.any?
+        puts Log.color(:magenta, "SACCT performance: ")
+        puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ")  + v.to_s } * "\n"
+      end
+    rescue
+      Log.warn $!.message
+    end
+  end
+
+
+  if tail && File.exist?(File.join(dir, 'std.err'))
+    if exit_status && exit_status != 0
+      puts Log.color(:magenta, "First error or exception found: ")
+      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+    elsif exit_status
+      puts Log.color(:magenta, "Completed jobs: ")
+      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+    else
+      puts Log.color(:magenta, "Log tail: ")
+      puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
+    end
+  end
+
+  if options[:progress] 
+    step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+    if step_line
+      require 'rbbt/workflow'
+      step_path = step_line.split(": ").last.strip
+      step = Step.new step_path
+      step.load_dependencies_from_info
+      has_bar = false
+      (step.rec_dependencies + [step]).reverse.each do |j|
+        next if j.done? 
+        if j.file(:progress).exists?
+          bar = Log::ProgressBar.new 
+          bar.load(j.file(:progress).yaml)
+          puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+          has_bar = true
+        end
+      end
+      step_status = step.status
+      step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
+      step_status = Log.color :green, step_status if step_status.to_s == 'done'
+      puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
+    end
+  end
+
+end
+
+puts 
+puts Log.color :clear, "Found #{count} jobs"
+

data/share/rbbt_commands/pbs/orchestrate

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+rbbt_options = SOPT::GOT_OPTIONS
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-a--account* Account
+-p--partition* Partition
+-t--task_cpus* Tasks
+-tm--time* Time
+-m--mem* SLURM minimum memory
+--gres* SLURM Generic resources
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
+-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
+-OR--orchestration_rules* Orchestration rules
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+$slurm_options[:config_keys] = SOPT::GOT_OPTIONS[:config_keys]
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+  def join
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+  end
+
+  def run(no_load = true)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing SLURM job for #{self.path}"
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
+        if no_load
+          self
+        else
+          join
+          self.load
+        end
+      rescue HPC::BATCH_DRY_RUN
+      end
+    end
+  end
+end
+
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/pbs/tail

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm tail <directory|jobid> [options] 
+
+-h--help Print this help
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+directory = ARGV.shift
+
+raise ParameterException if directory.nil?
+
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/rbbt-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
+
+require 'rbbt/hpc/slurm'
+
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Directory: ") + directory if directory
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
+HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/pbs/task

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
+require 'rbbt/hpc'
+require 'time'
+
+rbbt_options = SOPT::GOT_OPTIONS
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-a--account* Account
+-p--partition* Partition
+-t--task_cpus* Tasks
+-tm--time* Time
+-m--mem* minimum memory
+--gres* Generic resources
+-mcpu--mem_per_cpu* minimum memory per CPU
+-lin--licenses* licenses
+-cons--constraint* constraint
+-W--workflows* Additional workflows
+-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
+-OR--orchestration_rules* Orchestration rules
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing BATCH job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
+      rescue HPC::BATCH_DRY_RUN
+      end
+    end
+  end
+end
+
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+load Rbbt.share.rbbt_commands.workflow.task.find

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.40.0
+  version: 5.40.4
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-12 00:00:00.000000000 Z
+date: 2023-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -136,20 +136,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Utilities for handling tsv files, caches, etc
 email: miguel.vazquez.g@bsc.es
 executables:
@@ -302,6 +288,7 @@ files:
 - lib/rbbt/util/open.rb
 - lib/rbbt/util/procpath.rb
 - lib/rbbt/util/python.rb
+- lib/rbbt/util/python/step.rb
 - lib/rbbt/util/python/util.rb
 - lib/rbbt/util/semaphore.rb
 - lib/rbbt/util/simpleDSL.rb
@@ -390,6 +377,11 @@ files:
 - share/rbbt_commands/lsf/task
 - share/rbbt_commands/migrate
 - share/rbbt_commands/migrate_job
+- share/rbbt_commands/pbs/clean
+- share/rbbt_commands/pbs/list
+- share/rbbt_commands/pbs/orchestrate
+- share/rbbt_commands/pbs/tail
+- share/rbbt_commands/pbs/task
 - share/rbbt_commands/purge_job
 - share/rbbt_commands/resource/claims
 - share/rbbt_commands/resource/exists
@@ -592,7 +584,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.0.dev
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)