rbbt-util 5.28.12 → 5.29.4

data/lib/rbbt/persist.rb

@@ -110,6 +110,8 @@ module Persist
   def self.load_file(path, type)
     begin
       case (type || :marshal).to_sym
+      when :path
+        path
       when :nil
         nil
       when :boolean
@@ -167,6 +169,8 @@ module Persist
     end
 
     case (type || :marshal).to_sym
+    when :path
+      nil
     when :nil
       nil
     when :boolean

data/lib/rbbt/tsv/attach.rb

@@ -243,6 +243,7 @@ module TSV
     Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
 
     if complete
+      Log.warn "Attaching through index and completing empty rows; keys with wrong format may appear (#{other.key_field} insted of #{self.key_field})" if index
       fill = TrueClass === complete ? nil : complete
       field_length = self.fields.length 
       common_fields = (other.fields & self.fields)
@@ -255,11 +256,11 @@ module TSV
       case type
       when :single
         missing.each do |k|
-          self[k] = nil
+          self[k] = fill
         end
       when :list
         missing.each do |k|
-          values = [nil] * field_length
+          values = [fill] * field_length
           other_values = other[k]
           other_common_pos.zip(this_common_pos).each do |o,t|
             values[t] = other_values[o]
@@ -267,8 +268,9 @@ module TSV
           self[k] = values
         end
       when :double
+        fill = [] if fill.nil?
         missing.each do |k|
-          values = [[]] * field_length
+          values = [fill] * field_length
           other_values = other[k]
           other_common_pos.zip(this_common_pos).each do |o,t|
             values[t] = other_values[o]
@@ -276,8 +278,9 @@ module TSV
           self[k] = values
         end
       when :flat
+        fill = [] if fill.nil?
         missing.each do |k|
-          self[k] = []
+          self[k] = fill
         end
       end
     end

data/lib/rbbt/tsv/parallel.rb

@@ -2,6 +2,3 @@ require 'rbbt/util/concurrency'
 
 require 'rbbt/tsv/parallel/through'
 require 'rbbt/tsv/parallel/traverse'
-
-module TSV
-end

data/lib/rbbt/util/misc/inspect.rb

@@ -287,7 +287,9 @@ module Misc
           when Symbol 
             obj.to_s
           when (defined?(Path) and Path)
-            if Step === obj.resource
+            if defined?(Step) && Open.exists?(Step.info_file(obj))
+              obj2str(Workflow.load_step(obj))
+            elsif defined?(Step) && Step === obj.resource
               "Step file: " + obj
             else
               if obj.exists?
@@ -322,7 +324,11 @@ module Misc
             remove_long_items(obj)
           when File 
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
               "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
@@ -330,7 +336,11 @@ module Misc
             "<IO:" << obj.short_path << ">"
           when IO
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
 
               if obj.respond_to? :obj2str

data/lib/rbbt/workflow.rb

@@ -385,7 +385,7 @@ module Workflow
       next if default == v 
       next if (String === default and Symbol === v and v.to_s == default)
       next if (Symbol === default and String === v and v == default.to_s)
-      real_inputs[k] = v 
+      real_inputs[k.to_sym] = v 
     end
 
     jobname_input_value = inputs[jobname_input] || all_defaults[jobname_input]
@@ -410,6 +410,7 @@ module Workflow
     job.workflow = self
     job.clean_name = jobname
     job.overriden = overriden
+    job.real_inputs = real_inputs.keys
     job
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -16,6 +16,10 @@ end
 
 module Workflow
 
+  def self.job_path?(path)
+    path.split("/")[-4] == "jobs"
+  end
+
   def log(status, message = nil, &block)
     Step.log(status, message, nil, &block)
   end
@@ -301,7 +305,9 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep.workflow = workflow
     dep.info[:name] = dep.name
+    dep.original_task_name ||= dep.task_name if dep.workflow
     begin
       workflow = Kernel.const_get workflow if String === workflow
       dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
@@ -309,7 +315,7 @@ module Workflow
       Log.exception $!
     end
     dep.task_name = task_name
-    dep.overriden = true
+    dep.overriden = dep.original_task_name.to_sym
     dep
   end
 

data/lib/rbbt/workflow/definition.rb

@@ -77,6 +77,7 @@ module Workflow
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
       dep = dependencies.last.join
+      raise dep.get_exception if dep.error?
       set_info :result_type, dep.info[:result_type]
       forget = config :forget_dep_tasks, :forget_dep_tasks, :default => FORGET_DEP_TASKS
       if forget

data/lib/rbbt/workflow/examples.rb

@@ -50,8 +50,8 @@ module Workflow
         case input_types[input]
         when :file
           Log.debug "Pointing #{ input } to #{file}"
-          if file =~ /\.read$/
-            inputs[input.to_sym]  = Open.read(file)
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
           else
             inputs[input.to_sym]  = Open.realpath(file)
           end

data/lib/rbbt/workflow/step.rb

@@ -12,6 +12,7 @@ class Step
   attr_accessor :exec
   attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
+  attr_accessor :real_inputs, :original_task_name
 
   RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 
@@ -145,11 +146,13 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      dep_info[:inputs].each do |k,v|
-        all_inputs[k] = v unless all_inputs.include?(k)
-      end if dep_info[:inputs]
-      deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
-      deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      if dep_info
+        dep_info[:inputs].each do |k,v|
+          all_inputs[k] = v unless all_inputs.include?(k)
+        end if dep_info[:inputs]
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
       seen << path
     end
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -93,18 +93,22 @@ class Step
 
       Log.debug "Saving job input #{name} (#{type}) into #{path}"
       case
+      when Step === value
+        Open.ln_s(value.path, path)
+      when type.to_s == "file"
+        if String === value && File.exists?(value)
+          Open.ln_s(value, path)
+        else
+          Open.write(path + '.yaml', value.to_yaml)
+        end
       when Array === value
-        Open.write(path, value * "\n")
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
       when IO === value
-        Open.write(path, value)
-      when type == "file"
-        if String === value && File.exists?(value)
-          Open.link(value, path)
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
         else
-          Open.write(path + '.read', value.to_s)
+          Open.write(path, value)
         end
-      when Step === value
-        value = value.produce.load 
       else
         Open.write(path, value.to_s)
       end
@@ -114,18 +118,24 @@ class Step
   def self.save_job_inputs(job, dir, options = nil)
     options = IndiferentHash.setup options.dup if options
 
-    task_name = job.task_name
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
-    task_info = workflow.task_info(task_name)
-    input_types = task_info[:input_types]
-    task_inputs = task_info[:inputs]
-    input_defaults = task_info[:input_defaults]
+    if workflow
+      task_info = workflow.task_info(task_name)
+      input_types = task_info[:input_types]
+      task_inputs = task_info[:inputs]
+      input_defaults = task_info[:input_defaults]
+    else
+      task_info = input_types = task_inputs = input_defaults = {}
+    end
 
     inputs = {}
+    real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
-      next if options and ! options.include?(name)
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
       next if value.nil?
       next if input_defaults[name] == value
       inputs[name] = value
@@ -222,7 +232,7 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)

data/lib/rbbt/workflow/step/dependencies.rb

@@ -103,7 +103,7 @@ class Step
       end
 
       job.dup_inputs unless status == 'done' or job.started?
-      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
+      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started? || ! Workflow.job_path?(job.path)
 
       canfail = ComputeDependency === job && job.canfail?
     end
@@ -130,7 +130,6 @@ class Step
     (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
   end
 
-
   def execute_dependency(dependency, log = true)
     task_name = self.task_name
     canfail_paths = self.canfail_paths

data/lib/rbbt/workflow/step/run.rb

@@ -122,7 +122,6 @@ class Step
       reject{|dependency| (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dependency) || Open.remote?(dependency.path) }.
       reject{|dependency| dependency.error? }.
       #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
-      #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
       select{|dependency| dependency.updatable? }.
       collect{|dependency| Workflow.relocate_dependency(self, dependency)}
   end

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -26,15 +26,25 @@ module Workflow
       workload
     end
 
+    def self.workload(jobs)
+      jobs.inject({}) do |acc,job| 
+          Orchestrator.job_workload(job).each do |j,d|
+            acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
+          end
+          acc
+        end
+    end
+
     def self.job_rules(rules, job)
       workflow = job.workflow.to_s
       task_name = job.task_name.to_s
+      defaults = rules["defaults"] || {}
 
-      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow]
-      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow][task_name]
+      return IndiferentHash.setup(defaults) unless rules[workflow]
+      return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
 
       job_rules = IndiferentHash.setup(rules[workflow][task_name])
-      rules["defaults"].each{|k,v| job_rules[k] = v if job_rules[k].nil? } if rules["defaults"]
+      defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
       job_rules
     end
 
@@ -169,12 +179,7 @@ module Workflow
     def process(rules, jobs)
       begin
 
-        workload = jobs.inject({}) do |acc,job| 
-          Orchestrator.job_workload(job).each do |j,d|
-            acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
-          end
-          acc
-        end
+        workload = Orchestrator.workload(jobs)
         all_jobs = workload.keys
 
         top_level_jobs = jobs.collect{|job| job.path }

data/lib/rbbt/workflow/util/provenance.rb

@@ -22,11 +22,14 @@ class Step
   end
 
   def self.prov_report_msg(status, name, path, info = nil)
-    parts = path.sub(/\{.*/,'').sub(/#{Regexp.quote(name)}$/,'').split "/"
+    parts = path.sub(/\{.*/,'').split "/"
 
+    parts.pop
+    
     task = Log.color(:yellow, parts.pop)
     workflow = Log.color(:magenta, parts.pop)
-    if status.to_s == 'noinfo' and parts.last != 'jobs'
+    #if status.to_s == 'noinfo' && parts.last != 'jobs'
+    if ! Workflow.job_path?(path)
       task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
     end
 

data/share/rbbt_commands/slurm/clean

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean error or aborted jobs
+
+$ rbbt mnl [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-SBP--sbatch_parameters show sbatch parameters
+-dr--dry_run Do not erase anything
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+Log.severity = 4
+done, error, aborted, jobid, search, tail, sbatch_parameters, dry_run = options.values_at :done, :error, :aborted, :job, :search, :tail, :sbatch_parameters, :dry_run
+
+workdir = File.expand_path('~/rbbt-slurm')
+Path.setup(workdir)
+
+running_jobs = begin
+                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
+               end
+
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
+count = 0
+workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+ dir = File.dirname(fcmd)
+
+ if m = Open.read(fcmd).match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+  exe = m[1]
+ else
+  exe = nil
+ end
+
+ if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+
+ if File.exists?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'job.status'))
+  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exists?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+
+ aborted = error = true if aborted.nil? && error.nil?
+ if done || error || aborted || running || queued || jobid || search
+   select = false
+   select = true if done && exit_status && exit_status.to_i == 0 
+   select = true if error && exit_status && exit_status.to_i != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   select = select && jobid.split(",").include?(id) if jobid 
+   select = select && cmd.match(/#{search}/) if search
+   next unless select 
+ end
+
+
+ puts Log.color(:yellow, "**ERASING**")
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:sbatch_parameters]
+   puts Log.color(:magenta, "SBATCH parameters: ")
+   puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ end
+
+ if tail && File.exists?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
+
+ Open.rm_rf dir unless dry_run
+end
+
+puts 
+puts "Found #{count} jobs"
+