rbbt-util 5.30.9 → 5.31.0

data/share/rbbt_commands/{slurm → hpc}/list

@@ -2,6 +2,7 @@
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
 
 #$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -9,7 +10,7 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt mnl [options] 
+$ rbbt slurm list [options] 
 
 -h--help Print this help
 -d--done Done jobs only
@@ -20,9 +21,11 @@ $ rbbt mnl [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
+-p--progress Report progress of job and the dependencies
+-BP--batch_parameters show batch parameters
+-BPP--batch_procpath show Procpath performance summary
 -sacct--sacct_peformance show sacct performance summary
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]
@@ -34,14 +37,48 @@ if options[:help]
  exit 0
 end
 
-Log.severity = 4
-done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+batch_system = HPC::BATCH_MODULE.to_s.split("::").last.downcase
 
-workdir = File.expand_path('~/rbbt-slurm')
+done, error, running, queued, aborted, jobid, search, tail, progress = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail, :progress
+
+workdir = File.expand_path('~/rbbt-batch')
 Path.setup(workdir)
 
 running_jobs = begin
-                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt = HPC::BATCH_MODULE.job_status
                 squeue_txt.split("\n").collect{|l| l.to_i.to_s}
                rescue
                 Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
@@ -61,35 +98,48 @@ else
 end
 
 count = 0
-workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  dir = File.dirname(fcmd)
+ command_txt = Open.read(fcmd)
 
- if m = Open.read(fcmd).match(/#CMD: (.*)/)
+ if m = command_txt.match(/#CMD: (.*)/)
   cmd = m[1]
  else
   cmd = nil
  end
 
- if m = Open.read(fcmd).match(/#MANIFEST: (.*)/)
+ if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
+  job_batch_system = m[1].downcase
+ else
+  job_batch_system = nil
+ end
+
+ different_system = job_batch_system != batch_system
+
+ if m = command_txt.match(/#MANIFEST: (.*)/)
   manifest = m[1]
  else
   manifest = nil
  end
 
+ if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+ else
+  step_path = nil
+ end
 
- if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
-  exe = m[1].sub('step_path=$(','')
+ if m = command_txt.match(/#EXEC_CMD: (.*)/)
+  exe = m[1]
  else
   exe = nil
  end
 
- if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+ if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
   container_home = m[1]
  else
   container_home = nil
  end
 
-
  if File.exists?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
  else
@@ -103,11 +153,20 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  if File.exists?(fstatus = File.join(dir, 'job.status'))
-  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+   fstatus_txt = Open.read(fstatus) 
+   begin
+     if job_batch_system == "lsf"
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
+     else
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+     end
+   rescue
+     nodes = []
+   end
  elsif job_nodes[id]
-  nodes = job_nodes[id].reject{|n| n.include? "("}
+   nodes = job_nodes[id].reject{|n| n.include? "("}
  else
-  nodes = []
+   nodes = []
  end
 
  if File.exists?(File.join(dir, 'exit.status'))
@@ -136,7 +195,7 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
    select = true if done && exit_status == 0
    select = true if error && exit_status && exit_status != 0 
    select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
-   is_running =  exit_status.nil? && running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)
+   is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
@@ -150,29 +209,39 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
  puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
+ puts Log.color(:magenta, "Step path: ") << Log.color(:yellow, step_path)
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
  puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
  puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
  puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
- puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ if different_system
+   puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
+ else
+   puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ end
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", "
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
  puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
 
- if options[:sbatch_parameters]
-   puts Log.color(:magenta, "SBATCH parameters: ")
-   text = CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ if options[:batch_parameters]
+   puts Log.color(:magenta, "BATCH parameters: ")
+   case job_batch_system 
+   when 'slurm'
+     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+   when 'lsf'
+     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+   end
    lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
    puts Log.color :yellow, lines * "\n"
  end
 
  fprocpath = File.join(dir, 'procpath.sqlite3')
- if options[:procpath_performance]  && Open.exists?(fprocpath)
+ if options[:batch_procpath]  && Open.exists?(fprocpath)
    puts Log.color(:magenta, "Procpath summary: ")
    require 'rbbt/tsv/csv'
    meta = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from meta;' "))
@@ -214,13 +283,15 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
  if options[:sacct_peformance]
    begin
-    tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
-    values = tsv[tsv.keys.first]
-    if values.compact.any?
-     puts Log.color(:magenta, "SACCT performance: ")
-     puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ")  + v.to_s } * "\n"
-    end
+     raise "sacct not supported for LSF" unless batch_system == 'slurm' 
+     tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
+     values = tsv[tsv.keys.first]
+     if values.compact.any?
+       puts Log.color(:magenta, "SACCT performance: ")
+       puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ")  + v.to_s } * "\n"
+     end
    rescue
+     Log.warn $!.message
    end
  end
 
@@ -234,7 +305,24 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | tail -n #{tail.to_i} ").read
+   end
+ end
+
+ if options[:progress] 
+   step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+   if step_line
+     require 'rbbt/workflow'
+     step_path = step_line.split(": ").last.strip
+     step = Step.new step_path
+     step.load_dependencies_from_info
+     (step.rec_dependencies + [step]).reverse.each do |j|
+       next if j.done? 
+       next unless j.file(:progress).exists?
+       bar = Log::ProgressBar.new 
+       bar.load(j.file(:progress).yaml)
+       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+     end
    end
  end
 

data/share/rbbt_commands/hpc/orchestrate

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-OR--orchestration_rules* Orchestration rules
+-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/hpc/tail

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm tail <directory> [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-p--progress Report progress of job and the dependencies
+-SBP--sbatch_parameters show sbatch parameters
+-PERF--procpath_performance show Procpath performance summary
+-sacct--sacct_peformance show sacct performance summary
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+directory = ARGV.shift
+
+raise ParameterException if directory.nil?
+
+directory = File.dirname(directory) unless File.directory?(directory)
+
+require 'rbbt/hpc/slurm'
+
+HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/hpc/task

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find