rbbt-util 5.28.10 → 5.29.1

data/lib/rbbt/workflow/step/dependencies.rb

@@ -92,12 +92,18 @@ class Step
           (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
           (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
 
-        job.clean unless job.resumable? && (job.updated? && ! job.dirty?)
+        if ! (job.resumable? && (job.updated? && ! job.dirty?))
+          Log.high "About to clean -- status: #{status}, present #{File.exists?(job.path)}, " +
+            %w(done? error? recoverable_error? noinfo? updated? dirty? aborted? running? resumable?).
+            collect{|v| [v, job.send(v)]*": "} * ", " if RBBT_DEBUG_CLEAN
+
+          job.clean
+        end
         job.set_info :status, :cleaned
       end
 
       job.dup_inputs unless status == 'done' or job.started?
-      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
+      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started? || ! Workflow.job_path?(job.path)
 
       canfail = ComputeDependency === job && job.canfail?
     end
@@ -121,10 +127,9 @@ class Step
   end
 
   def input_dependencies
-    inputs.flatten.select{|i| Step === i}
+    (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
   end
 
-
   def execute_dependency(dependency, log = true)
     task_name = self.task_name
     canfail_paths = self.canfail_paths

data/lib/rbbt/workflow/step/run.rb

@@ -112,7 +112,7 @@ class Step
   end
 
   def updatable?
-    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?))
+    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?)) || (ENV["RBBT_UPDATE"] && ! (done? && ! Open.exists?(info_file)))
   end
 
   def dependency_checks
@@ -122,13 +122,12 @@ class Step
       reject{|dependency| (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dependency) || Open.remote?(dependency.path) }.
       reject{|dependency| dependency.error? }.
       #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
-      #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
       select{|dependency| dependency.updatable? }.
       collect{|dependency| Workflow.relocate_dependency(self, dependency)}
   end
 
   def input_checks
-    inputs.select{|i| Step === i }.
+    (inputs.select{|i| Step === i } + inputs.select{|i| Path === i && Step === i.resource}.collect{|i| i.resource}).
       select{|dependency| dependency.updatable? }
   end
 
@@ -154,25 +153,28 @@ class Step
     canfail_paths = self.canfail_paths
     this_mtime = Open.mtime(self.path) if Open.exists?(self.path)
 
-    checks.each do |dep| 
-      next unless dep.updatable?
-      dep_done = dep.done?
+    outdated_time = checks.select{|dep| dep.updatable? && dep.done? && Persist.newer?(path, dep.path) }
+    outdated_dep = checks.reject{|dep| dep.done? || (dep.error? && ! dep.recoverable_error? && canfail_paths.include?(dep.path)) }
 
-      begin
-        if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
-          outdated_time << dep
-        end
-      rescue
-      end
+    #checks.each do |dep| 
+    #  next unless dep.updatable?
+    #  dep_done = dep.done?
 
-      # Is this pointless? this would mean some dep got updated after a later
-      # dep but but before this one.
-      #if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
+    #  begin
+    #    if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
+    #      outdated_time << dep
+    #    end
+    #  rescue
+    #  end
 
-      if (! dep_done && ! canfail_paths.include?(dep.path))
-        outdated_dep << dep
-      end
-    end
+    #  # Is this pointless? this would mean some dep got updated after a later
+    #  # dep but but before this one.
+    #  #if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
+
+    #  if (! dep_done && ! canfail_paths.include?(dep.path))
+    #    outdated_dep << dep
+    #  end
+    #end
 
     Log.high "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
     Log.high "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
@@ -215,7 +217,7 @@ class Step
         no_load = :stream if no_load
 
         Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result_type = @task.result_type
+        result_type = @task.result_type if @task
         result_type = info[:result_type] if result_type.nil?
         result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -26,15 +26,25 @@ module Workflow
       workload
     end
 
+    def self.workload(jobs)
+      jobs.inject({}) do |acc,job| 
+          Orchestrator.job_workload(job).each do |j,d|
+            acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
+          end
+          acc
+        end
+    end
+
     def self.job_rules(rules, job)
       workflow = job.workflow.to_s
       task_name = job.task_name.to_s
+      defaults = rules["defaults"] || {}
 
-      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow]
-      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow][task_name]
+      return IndiferentHash.setup(defaults) unless rules[workflow]
+      return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
 
       job_rules = IndiferentHash.setup(rules[workflow][task_name])
-      rules["defaults"].each{|k,v| job_rules[k] = v if job_rules[k].nil? } if rules["defaults"]
+      defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
       job_rules
     end
 
@@ -169,12 +179,7 @@ module Workflow
     def process(rules, jobs)
       begin
 
-        workload = jobs.inject({}) do |acc,job| 
-          Orchestrator.job_workload(job).each do |j,d|
-            acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
-          end
-          acc
-        end
+        workload = Orchestrator.workload(jobs)
         all_jobs = workload.keys
 
         top_level_jobs = jobs.collect{|job| job.path }

data/lib/rbbt/workflow/util/provenance.rb

@@ -22,11 +22,14 @@ class Step
   end
 
   def self.prov_report_msg(status, name, path, info = nil)
-    parts = path.sub(/\{.*/,'').sub(/#{Regexp.quote(name)}$/,'').split "/"
+    parts = path.sub(/\{.*/,'').split "/"
 
+    parts.pop
+    
     task = Log.color(:yellow, parts.pop)
     workflow = Log.color(:magenta, parts.pop)
-    if status.to_s == 'noinfo' and parts.last != 'jobs'
+    #if status.to_s == 'noinfo' && parts.last != 'jobs'
+    if ! Workflow.job_path?(path)
       task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
     end
 
@@ -66,7 +69,7 @@ class Step
     str << "\n"
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [])
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -82,9 +85,13 @@ class Step
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen)
+        str << prov_report(dep, offset + 1, task, seen, expand_repeats)
       else
-        str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+        if expand_repeats
+          str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+        else
+          str << Log.color(:green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+        end
       end
     end if step.dependencies
     str

data/share/rbbt_commands/slurm/list

@@ -0,0 +1,141 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt mnl [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+EOF
+
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+Log.severity = 4
+done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
+
+workdir = File.expand_path('~/rbbt-slurm')
+Path.setup(workdir)
+
+running_jobs = begin
+                   CMD.cmd('squeue').read.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                   Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                   $norunningjobs = true
+                   []
+               end
+
+count = 0
+workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+    dir = File.dirname(fcmd)
+
+    if m = Open.read(fcmd).match(/#CMD: (.*)/)
+        cmd = m[1]
+    else
+        cmd = nil
+    end
+
+    if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+        exe = m[1]
+    else
+        exe = nil
+    end
+
+    if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+        container_home = m[1]
+    else
+        container_home = nil
+    end
+
+
+    if File.exists?(fid = File.join(dir, 'job.id'))
+        id = Open.read(fid).chomp
+    else
+        id = nil
+    end
+
+    if File.exists?(fstatus = File.join(dir, 'exit.status'))
+        exit_status = Open.read(fstatus).to_i
+    else
+        exit_status = nil
+    end
+    
+    if File.exists?(fstatus = File.join(dir, 'job.status'))
+        nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+    else
+        nodes = []
+    end
+
+    if File.exists?(File.join(dir, 'std.out'))
+        outt = File.mtime File.join(dir, 'std.out')
+        errt = File.mtime File.join(dir, 'std.err')
+        time_diff = Time.now - [outt, errt].max
+    end
+
+    fdep = File.join(dir, 'dependencies.list')
+    deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+    if done || error || aborted || running || queued || jobid || search
+      select = false
+      select = true if done && exit_status == 0
+      select = true if error && exit_status && exit_status != 0 
+      select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+      select = true if queued && deps && (running_jobs & deps).any?
+      select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
+      select = true if jobid && jobid.split(",").include?(id)
+      select = true if search && cmd.match(/#{search}/)
+      next unless select 
+    end
+
+
+    puts Log.color :blue, dir
+    puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+    puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+    puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+    puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+    puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+    puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+    puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+    puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+    if tail && File.exists?(File.join(dir, 'std.err'))
+        if exit_status && exit_status != 0
+            puts Log.color(:magenta, "First error or exception found: ")
+            puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+        elsif exit_status
+            puts Log.color(:magenta, "Completed jobs: ")
+            puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+        else
+            puts Log.color(:magenta, "Log tail: ")
+            puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+        end
+    end
+
+    count += 1
+
+end
+
+puts 
+puts "Found #{count} jobs"
+

data/share/rbbt_commands/slurm/orchestrate

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-R--rules* Orchestration rules
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/{workflow/slurm → slurm/task}

@@ -9,8 +9,9 @@ require 'time'
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
---drbbt Use development version of rbbt 
+--drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
 -e--exclusive Make exclusive use of the node
@@ -21,8 +22,9 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -q--queue* Queue
 -t--task_cpus* Tasks
+-W--workflows* Additional workflows
 -tm--time* Time
--S--server* SLURM login node
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 
 class Step
@@ -30,9 +32,14 @@ class Step
         if done?
             self.load
         else
-            Marenostrum::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
         end
     end
 end
 
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/system/status

@@ -49,27 +49,27 @@ def pid_msg(pid)
 end
 
 
-def status_msg(status)
-  color = case status.to_sym
-          when :error, :aborted, :missing, :dead, :broken
-            :red
-          when :streaming, :started
-            :cyan
-          when :done
-            :green
-          when :noinfo
-            :blue
-          when :dependencies, :waiting, :setup
-            :yellow
-          else
-            if status.to_s.index ">"
-              :cyan
-            else
-              :cyan
-            end
-          end
-  Log.color(color, status.to_s)
-end
+#def status_msg(status)
+#  color = case status.to_sym
+#          when :error, :aborted, :missing, :dead, :broken
+#            :red
+#          when :streaming, :started
+#            :cyan
+#          when :done
+#            :green
+#          when :noinfo, :notfound
+#            :blue
+#          when :dependencies, :waiting, :setup
+#            :yellow
+#          else
+#            if status.to_s.index ">"
+#              :cyan
+#            else
+#              :cyan
+#            end
+#          end
+#  Log.color(color, status.to_s)
+#end
 
 def input_msg(file, inputs)
 
@@ -218,7 +218,7 @@ workflows.sort.each do |workflow,tasks|
             status << Log.color(:red, " (dead)") 
           end
         end
-        str << " #{ status_msg status }"
+        str << " #{ Step.prov_status_msg status }"
         str << " (dirty)" if status == 'done' && Workflow.load_step(file).dirty?
 
         if inputs and inputs.any?