rbbt-util 5.29.0 → 5.30.1

data/share/rbbt_commands/slurm/list

@@ -20,15 +20,18 @@ $ rbbt mnl [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-SBP--sbatch_parameters show sbatch parameters
+-PERF--procpath_performance show Procpath performance summary
+-sacct--sacct_peformance show sacct performance summary
 EOF
 
 if options[:help]
-  if defined? rbbt_usage
-    rbbt_usage 
-  else
-    puts SOPT.doc
-  end
-  exit 0
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
 end
 
 Log.severity = 4
@@ -38,101 +41,177 @@ workdir = File.expand_path('~/rbbt-slurm')
 Path.setup(workdir)
 
 running_jobs = begin
-                   CMD.cmd('squeue').read.split("\n").collect{|l| l.to_i.to_s}
+                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
                rescue
-                   Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
-                   $norunningjobs = true
-                   []
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
                end
 
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
 count = 0
 workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
-    dir = File.dirname(fcmd)
-
-    if m = Open.read(fcmd).match(/#CMD: (.*)/)
-        cmd = m[1]
-    else
-        cmd = nil
+ dir = File.dirname(fcmd)
+
+ if m = Open.read(fcmd).match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+  exe = m[1].sub('step_path=$(','')
+ else
+  exe = nil
+ end
+
+ if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+
+ if File.exists?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'job.status'))
+  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exists?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+
+ if done || error || aborted || running || queued || jobid || search
+   select = false
+   select = true if done && exit_status == 0
+   select = true if error && exit_status && exit_status != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   select = true if queued && deps && (running_jobs & deps).any?
+   select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
+   select = true if jobid && jobid.split(",").include?(id)
+   select = true if search && cmd.match(/#{search}/)
+   next unless select 
+ end
+
+
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:sbatch_parameters]
+   puts Log.color(:magenta, "SBATCH parameters: ")
+   text = CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
+   puts Log.color :yellow, lines * "\n"
+ end
+
+ fprocpath = File.join(dir, 'procpath.sqlite3')
+ if options[:procpath_performance]  && Open.exists?(fprocpath)
+   puts Log.color(:magenta, "Procpath summary: ")
+   require 'rbbt/tsv/csv'
+   meta = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from meta;' "))
+   perf = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from record;' "))
+
+   page_size = meta["page_size"].first.to_f
+   clock_ticks = meta["clock_ticks"].first.to_f
+
+   cpu_average = {}
+   rss_average = {}
+   perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
+    time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
+    time = time.to_f
+
+    cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
+    cpu_average[stat_pid] ||= {}
+    cpu_average[stat_pid][time] ||= []
+    cpu_average[stat_pid][time] << cpu.to_f
+    rss_average[time] ||= []
+    rss_average[time] << rss.to_f * page_size
+   end
+
+   ticks = 0
+   cpu_average.each do |stat_pid, cpu_average_pid|
+    start = cpu_average_pid.keys.sort.first
+    eend = cpu_average_pid.keys.sort.last
+    ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
+   end
+   start = rss_average.keys.sort.first
+   eend = rss_average.keys.sort.last
+   time_elapsed = eend - start
+   puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
+   puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s 
+
+ end
+
+ if options[:sacct_peformance]
+   begin
+    tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
+    values = tsv[tsv.keys.first]
+    if values.compact.any?
+     puts Log.color(:magenta, "SACCT performance: ")
+     puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ")  + v.to_s } * "\n"
     end
-
-    if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
-        exe = m[1]
-    else
-        exe = nil
-    end
-
-    if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
-        container_home = m[1]
-    else
-        container_home = nil
-    end
-
-
-    if File.exists?(fid = File.join(dir, 'job.id'))
-        id = Open.read(fid).chomp
-    else
-        id = nil
-    end
-
-    if File.exists?(fstatus = File.join(dir, 'exit.status'))
-        exit_status = Open.read(fstatus).to_i
-    else
-        exit_status = nil
-    end
-    
-    if File.exists?(fstatus = File.join(dir, 'job.status'))
-        nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
-    else
-        nodes = []
-    end
-
-    if File.exists?(File.join(dir, 'std.out'))
-        outt = File.mtime File.join(dir, 'std.out')
-        errt = File.mtime File.join(dir, 'std.err')
-        time_diff = Time.now - [outt, errt].max
-    end
-
-    fdep = File.join(dir, 'dependencies.list')
-    deps = Open.read(fdep).split("\n") if File.exists?(fdep)
-
-    if done || error || aborted || running || queued || jobid || search
-      select = false
-      select = true if done && exit_status == 0
-      select = true if error && exit_status && exit_status != 0 
-      select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
-      select = true if queued && deps && (running_jobs & deps).any?
-      select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
-      select = true if jobid && jobid.split(",").include?(id)
-      select = true if search && cmd.match(/#{search}/)
-      next unless select 
-    end
-
-
-    puts Log.color :blue, dir
-    puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
-    puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
-    puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
-    puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
-    puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
-    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
-    puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
-    puts Log.color(:magenta, "Nodes: ") << nodes * ", "
-    puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
-
-    if tail && File.exists?(File.join(dir, 'std.err'))
-        if exit_status && exit_status != 0
-            puts Log.color(:magenta, "First error or exception found: ")
-            puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
-        elsif exit_status
-            puts Log.color(:magenta, "Completed jobs: ")
-            puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
-        else
-            puts Log.color(:magenta, "Log tail: ")
-            puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
-        end
-    end
-
-    count += 1
+   rescue
+   end
+ end
+
+
+ if tail && File.exists?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
 
 end
 

data/share/rbbt_commands/slurm/orchestrate

@@ -21,11 +21,12 @@ $slurm_options = SOPT.get <<EOF
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
+-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
 -W--workflows* Additional workflows
 -tm--time* Time
--R--rules* Orchestration rules
+-OR--orchestration_rules* Orchestration rules
 -rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 
@@ -43,5 +44,5 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/task

@@ -20,6 +20,7 @@ $slurm_options = SOPT.get <<EOF
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
+-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
 -W--workflows* Additional workflows

data/share/rbbt_commands/tsv/slice

@@ -35,7 +35,7 @@ file = case file
 fields = options[:fields]
 raise ParameterException, "Please specify the fields to slice" if fields.nil?
 
-options[:header_hash] = options["header_hash"]
+options[:header_hash] ||= options["header_hash"]
 
 case
 when options[:tokyocabinet]
@@ -45,8 +45,8 @@ when options[:tokyocabinet_bd]
   tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
   puts tsv.summary
 else
-  stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :keys => fields, :unnamed => true) do |*p|
-    p * "\t"
+  stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
+    [k,fields].flatten * "\t"
   end
   puts stream.read
   exit 0

data/share/rbbt_commands/workflow/info

@@ -86,7 +86,7 @@ messages = info[:messages]
 backtrace = info[:backtrace]
 pid = info[:pid]
 exception = info[:exception]
-rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
+rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :archived_info]
 
 
 puts Log.color(:magenta, "File") << ": " << step.path

data/share/rbbt_commands/workflow/task

@@ -20,7 +20,7 @@ def usage(workflow = nil, task = nil, exception=nil, abridge = false)
   puts
   if workflow.nil?
     puts "No workflow specified. Use `rbbt workflow list` to list available workflows."
-    exit -1 
+    exit! -1 
   end
 
   if task.nil?
@@ -203,10 +203,11 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -prec--prepare_cpus* Number of dependencies prepared in parallel
 -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
 -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
+-PERF--procpath_performance* Measure performance using procpath
 EOF
 
 workflow = ARGV.shift
-usage and exit -1 if workflow.nil?
+usage and exit! -1 if workflow.nil?
 
 task     = ARGV.shift
 
@@ -232,7 +233,8 @@ else
   remote_workflows = {}
 end
 
-Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+#Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+Workflow.workdir.search_paths.merge!({:workdir => File.expand_path(options.delete(:workdir_all)), :default => :workdir }) if options[:workdir_all]
 
 workflow = Workflow.require_workflow workflow
 
@@ -406,6 +408,23 @@ begin
     exit 0
   end
 
+  if options[:procpath_performance]
+    require 'rbbt/util/procpath'
+    current_pid = job.info[:pid]
+    job.fork
+    job.soft_grace
+    sleep 2 if job.info[:pid] == current_pid
+    if job.info[:pid] != current_pid
+      pid = job.info[:pid]
+      begin
+        ProcPath.monitor(pid, options[:procpath_performance])
+      rescue Errno::ECHILD
+        Log.warn "Procpath didn't find process #{pid} to monitor. Maybe it finished already"
+      rescue
+        Log.warn "Procpath failed: #{$!.message}"
+      end
+    end
+  end
 
   if do_fork
     ENV["RBBT_NO_PROGRESS"] = "true"
@@ -422,7 +441,6 @@ begin
     res = job
   end
 
-
   if options.delete(:printpath)
     job.join
     raise job.messages.last if (job.error? || job.aborted?) && job.messages
@@ -486,7 +504,7 @@ rescue ParameterException
   puts
   report_options saved_job_options
   puts
-  exit -1
+  exit! -1
 end
 
 if options.delete(:list_job_files)
@@ -538,7 +556,7 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     rescue Exception
       Log.exception $!
@@ -547,9 +565,11 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     end
+  elsif detach
+    exit! 0
   else
     res.join
     out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

data/share/rbbt_commands/workflow/write_info

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/workflow'
+
+require 'rbbt-util'
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Examine the info of a job result
+
+$ rbbt workflow info <job-result> <key> <value>
+
+-h--help Help
+-f--force Write info even if key is already present
+-r--recursive Write info for all dependencies as well
+-p--check_pid Check that recursive jobs where created by the same process
+EOF
+
+SOPT.usage if options[:help]
+
+file, key, value = ARGV
+
+force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
+
+def get_step(file)
+  file = file.sub(/\.(info|files)/,'')
+  step = Workflow.load_step file
+  step
+end
+
+raise ParameterException if key.nil? || value.nil?
+
+if %w(DELETE nil).include? value
+  value = nil
+  force = true
+end
+
+step = get_step file
+
+step.set_info key, value if force || ! step.info.include?(key)
+
+pid = step.info[:pid]
+host = step.info[:pid_hostname]
+
+step.rec_dependencies.each do |dep|
+  dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
+rescue
+  Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
+end if recursive