rbbt-util 5.28.9 → 5.29.0

data/lib/rbbt/workflow/util/provenance.rb

@@ -66,7 +66,7 @@ class Step
     str << "\n"
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [])
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -82,9 +82,13 @@ class Step
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen)
+        str << prov_report(dep, offset + 1, task, seen, expand_repeats)
       else
-        str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+        if expand_repeats
+          str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+        else
+          str << Log.color(:green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+        end
       end
     end if step.dependencies
     str

data/share/rbbt_commands/ansible

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Run ansible
+
+$ #{$0} [options] <rules.yml|->
+
+Use - to read from STDIN
+
+-h--help Print this help
+-dr--dry_run Only print the playbook
+-H--hosts* hosts to run it on
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+require 'rbbt/workflow/integration/ansible'
+
+playbook, task = ARGV
+playbook_file = Rbbt.share.ansible["test.yaml"].find if playbook.nil?
+
+dry_run = options.delete :dry_run
+
+if ! Misc.filename?(playbook_file)
+  playbook_file = Rbbt.share.ansible.glob(playbook + ".*").first
+end
+
+if ! Misc.filename?(playbook_file)
+  recipe = Rbbt.share.recipes.glob(playbook + ".*").first
+  playbook_file = Ansible.playbook recipe, task, options
+end
+
+
+if dry_run
+  if Hash === playbook_file
+    ppp playbook_file.to_yaml
+  else
+    ppp Log.color :magenta, playbook_file
+    ppp Open.read(playbook_file) 
+  end
+else
+  Ansible.play playbook_file
+end

data/share/rbbt_commands/purge_job

@@ -14,7 +14,6 @@ $ rbbt purge [options] <job_path>
 
 -h--help Print this help
 -r--recursive Remove recursively
-
 EOF
 if options[:help]
   if defined? rbbt_usage

data/share/rbbt_commands/slurm/list

@@ -0,0 +1,141 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt mnl [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+EOF
+
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+Log.severity = 4
+done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
+
+workdir = File.expand_path('~/rbbt-slurm')
+Path.setup(workdir)
+
+running_jobs = begin
+                   CMD.cmd('squeue').read.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                   Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                   $norunningjobs = true
+                   []
+               end
+
+count = 0
+workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+    dir = File.dirname(fcmd)
+
+    if m = Open.read(fcmd).match(/#CMD: (.*)/)
+        cmd = m[1]
+    else
+        cmd = nil
+    end
+
+    if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+        exe = m[1]
+    else
+        exe = nil
+    end
+
+    if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+        container_home = m[1]
+    else
+        container_home = nil
+    end
+
+
+    if File.exists?(fid = File.join(dir, 'job.id'))
+        id = Open.read(fid).chomp
+    else
+        id = nil
+    end
+
+    if File.exists?(fstatus = File.join(dir, 'exit.status'))
+        exit_status = Open.read(fstatus).to_i
+    else
+        exit_status = nil
+    end
+    
+    if File.exists?(fstatus = File.join(dir, 'job.status'))
+        nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+    else
+        nodes = []
+    end
+
+    if File.exists?(File.join(dir, 'std.out'))
+        outt = File.mtime File.join(dir, 'std.out')
+        errt = File.mtime File.join(dir, 'std.err')
+        time_diff = Time.now - [outt, errt].max
+    end
+
+    fdep = File.join(dir, 'dependencies.list')
+    deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+    if done || error || aborted || running || queued || jobid || search
+      select = false
+      select = true if done && exit_status == 0
+      select = true if error && exit_status && exit_status != 0 
+      select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+      select = true if queued && deps && (running_jobs & deps).any?
+      select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
+      select = true if jobid && jobid.split(",").include?(id)
+      select = true if search && cmd.match(/#{search}/)
+      next unless select 
+    end
+
+
+    puts Log.color :blue, dir
+    puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+    puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+    puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+    puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+    puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+    puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+    puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+    puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+    if tail && File.exists?(File.join(dir, 'std.err'))
+        if exit_status && exit_status != 0
+            puts Log.color(:magenta, "First error or exception found: ")
+            puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+        elsif exit_status
+            puts Log.color(:magenta, "Completed jobs: ")
+            puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+        else
+            puts Log.color(:magenta, "Log tail: ")
+            puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+        end
+    end
+
+    count += 1
+
+end
+
+puts 
+puts "Found #{count} jobs"
+

data/share/rbbt_commands/slurm/orchestrate

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-R--rules* Orchestration rules
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/{workflow/slurm → slurm/task}

@@ -9,8 +9,9 @@ require 'time'
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
---drbbt Use development version of rbbt 
+--drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
 -e--exclusive Make exclusive use of the node
@@ -21,8 +22,9 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -q--queue* Queue
 -t--task_cpus* Tasks
+-W--workflows* Additional workflows
 -tm--time* Time
--S--server* SLURM login node
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 
 class Step
@@ -30,9 +32,14 @@ class Step
         if done?
             self.load
         else
-            Marenostrum::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
         end
     end
 end
 
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/system/status

@@ -49,27 +49,27 @@ def pid_msg(pid)
 end
 
 
-def status_msg(status)
-  color = case status.to_sym
-          when :error, :aborted, :missing, :dead, :broken
-            :red
-          when :streaming, :started
-            :cyan
-          when :done
-            :green
-          when :noinfo
-            :blue
-          when :dependencies, :waiting, :setup
-            :yellow
-          else
-            if status.to_s.index ">"
-              :cyan
-            else
-              :cyan
-            end
-          end
-  Log.color(color, status.to_s)
-end
+#def status_msg(status)
+#  color = case status.to_sym
+#          when :error, :aborted, :missing, :dead, :broken
+#            :red
+#          when :streaming, :started
+#            :cyan
+#          when :done
+#            :green
+#          when :noinfo, :notfound
+#            :blue
+#          when :dependencies, :waiting, :setup
+#            :yellow
+#          else
+#            if status.to_s.index ">"
+#              :cyan
+#            else
+#              :cyan
+#            end
+#          end
+#  Log.color(color, status.to_s)
+#end
 
 def input_msg(file, inputs)
 
@@ -218,7 +218,7 @@ workflows.sort.each do |workflow,tasks|
             status << Log.color(:red, " (dead)") 
           end
         end
-        str << " #{ status_msg status }"
+        str << " #{ Step.prov_status_msg status }"
         str << " (dirty)" if status == 'done' && Workflow.load_step(file).dirty?
 
         if inputs and inputs.any? 

data/share/rbbt_commands/workflow/forget_deps

@@ -13,6 +13,8 @@ Make a job forget all its dependencies and archive their meta-data
 $ #{$0} [options] <job_path>
 
 -h--help Print this help
+-p--purge Purge dependencies
+-r--recursive_purge Purge dependencies recursively
 
 EOF
 if options[:help]
@@ -31,4 +33,11 @@ step = Workflow.load_step path
 
 step.archive_deps
 step.copy_files_dir
+dependencies = step.dependencies
 step.set_info :dependencies, []
+
+if options[:purge]
+  dependencies.each do |dependency|
+    Step.purge(dependency.path, options[:recursive_purge])
+  end
+end

data/share/rbbt_commands/workflow/info

@@ -17,6 +17,7 @@ $ rbbt workflow info <job-result>
 -a--all Print all info entries
 -r--recursive Print recursive input values
 -o--original Print original object
+-w--width* Screen width
 EOF
 
 SOPT.usage if options[:help]
@@ -24,6 +25,7 @@ SOPT.usage if options[:help]
 file = ARGV.shift
 all = options.delete :all
 recursive = options.delete :recursive
+width = (options.delete(:width) || 80).to_i
 
 def get_step(file)
   file = file.sub(/\.(info|files)/,'')
@@ -86,6 +88,7 @@ pid = info[:pid]
 exception = info[:exception]
 rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
 
+
 puts Log.color(:magenta, "File") << ": " << step.path
 puts Log.color(:magenta, "Status") << ": " << status_msg(status) << ((step.aborted? || step.error?) && step.recoverable_error? ? " (recoverable)" : "" ) << (step.dirty? ? " (dirty)" : "")
 puts Log.color(:magenta, "Pid") << ": " << pid_msg(pid, status.to_s == "done")
@@ -96,14 +99,14 @@ if inputs and inputs.any?
   inputs.each do |input,value|
     case value
     when nil
-      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', width, 20, :blue)
     when Array
-      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), width, 20, :blue)
     when TrueClass, FalseClass
-      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, width, 20, :blue)
     else
       text = value.to_s.split("\n")[0..5].compact * "\n\n"
-      puts Misc.format_definition_list_item("  " + input.to_s, text, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, text, width, 20, :blue)
     end
   end
 end
@@ -162,16 +165,16 @@ if recursive
     inputs.each do |input,value|
       case value
       when nil
-        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', 80, 20, :blue)
+        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', width, 20, :blue)
       when Array
-        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), 80, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), width, 20, :blue).gsub("\n\n","\n")
       when TrueClass, FalseClass
-        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, 80, 20, :blue)
+        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, width, 20, :blue)
       else
-        lines = value.to_s.split("\n").collect{|l| l.length >= 60 ? l[0..45] + " ..." : l }
+        lines = value.to_s.split("\n").collect{|l| l.length >= width - 5 ? l[0..width - 5] + " ..." : l }
         text = lines[0..5].compact * "\n\n"
         text << "\n\n...\n\n" if lines.length > 6
-        puts Misc.format_definition_list_item("  " << input.to_s, text, 80, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " << input.to_s, text, width, 20, :blue).gsub("\n\n","\n")
       end
     end
   end