rbbt-util 5.29.0 → 5.30.1

data/lib/rbbt/tsv.rb

@@ -113,8 +113,8 @@ module TSV
 
     data.entity_options = entity_options
 
-    if Path === source and data.identifiers
-      data.identifiers = Path.setup(data.identifiers, source.pkgdir, source.resource)
+    if Path === source && data.identifiers
+      Path.setup(data.identifiers, source.pkgdir, source.resource)
     end
 
     if data.respond_to? :persistence_path
@@ -124,6 +124,10 @@ module TSV
       data.clear
       data.annotate h
     end
+
+    data.read 
+
+    data
   end
 
   def self.parse_header(stream, options = {})

data/lib/rbbt/util/cmd.rb

@@ -217,7 +217,7 @@ module CMD
     end
   end
 
-  def self.cmd_log(*args)
+  def self.cmd_pid(*args)
     all_args = *args
 
     all_args << {} unless Hash === all_args.last
@@ -248,4 +248,9 @@ module CMD
     nil
   end
 
+  def self.cmd_log(*args)
+    cmd_pid(*args)
+    nil
+  end
+
 end

data/lib/rbbt/util/misc/inspect.rb

@@ -287,7 +287,9 @@ module Misc
           when Symbol 
             obj.to_s
           when (defined?(Path) and Path)
-            if Step === obj.resource
+            if defined?(Step) && Open.exists?(Step.info_file(obj))
+              obj2str(Workflow.load_step(obj))
+            elsif defined?(Step) && Step === obj.resource
               "Step file: " + obj
             else
               if obj.exists?
@@ -322,7 +324,11 @@ module Misc
             remove_long_items(obj)
           when File 
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
               "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
@@ -330,7 +336,11 @@ module Misc
             "<IO:" << obj.short_path << ">"
           when IO
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
 
               if obj.respond_to? :obj2str

data/lib/rbbt/util/misc/options.rb

@@ -242,48 +242,6 @@ module Misc
 
     return options
 
-    options = {}
-    string.split(/#/).each do |str|
-      if str.match(/(.*)=(.*)/)
-        option, value = $1, $2
-      else
-        option, value = str, true
-      end
-
-      option = option.sub(":",'').to_sym if option.chars.first == ':'
-      value  = value.sub(":",'').to_sym if String === value and value.chars.first == ':'
-
-      if value == true
-        options[option] = option.to_s.chars.first != '!' 
-      else
-        options[option] = Thread.start do
-          $SAFE = 0;
-          case 
-          when value =~ /^(?:true|T)$/i
-            true
-          when value =~ /^(?:false|F)$/i
-            false
-          when Symbol === value
-            value
-          when (String === value and value =~ /^\/(.*)\/$/)
-            Regexp.new /#{$1}/
-          else
-            begin
-              Kernel.const_get value
-            rescue
-              begin  
-                raise if value =~ /[a-z]/ and defined? value
-                eval(value) 
-              rescue Exception
-                value 
-              end
-            end
-          end
-        end.value
-      end
-    end
-
-    options
   end
 
 end

data/lib/rbbt/util/procpath.rb

@@ -0,0 +1,49 @@
+require 'rbbt/util/cmd'
+module ProcPath
+  CMD.tool :procpath do
+    'pip install procpath'
+  end
+
+  def self.record(pid, path, options = {})
+    IndiferentHash.setup(options)
+    options = Misc.add_defaults options, "interval" => 30
+
+    cmd_options = %w(interval recnum reevalnum).inject({}){|acc,k| acc[k] = options[k]; acc}
+
+    Log.debug "ProcPath recording #{pid} in #{path} (#{Misc.fingerprint options})"
+    procpath_thread = Thread.new do 
+      begin
+        procpath_pid = CMD.cmd_pid(:procpath, "record --database-file '#{path}' '$..children[?(@.stat.pid == #{pid})]'", cmd_options.merge(:nofail => true, :add_option_dashes => true))
+      rescue Exception
+        Log.exceptions $!
+        Process.kill "INT", procpath_pid
+      end
+    end
+
+    procpath_thread.report_on_exception = false
+
+    Process.wait pid.to_i
+    procpath_thread.raise Interrupt
+  end
+
+  def self.plot(path, output, options = {})
+    IndiferentHash.setup(options)
+    options = Misc.add_defaults options, "query-name" => 'rss', 'epsilon' => 0.5, "moving-average-window" => 10
+
+    cmd_options = %w(query-name epsilon monitor-average-window title logarithmic after before custom-query-file custom-value-expr).inject({}){|acc,k| acc[k] = options[k]; acc}
+    CMD.cmd_log(:procpath, "plot --database-file '#{path}' --plot-file '#{output}' ", cmd_options.merge(:nofail => true, :add_option_dashes => true))
+  end
+
+  def self.monitor(pid, path)
+    database, options_str = path.split("#")
+    options = options_str.nil? ? {} : Misc.string2hash(options_str)
+
+    database = File.expand_path database
+    Log.low "ProcPath monitor #{pid} in #{database} (#{Misc.fingerprint options})"
+
+    ProcPath.record(pid, database + '.sqlite3', options)
+    ProcPath.plot(database + '.sqlite3', database + '.cpu.svg', options.merge("query-name" => 'cpu'))
+    ProcPath.plot(database + '.sqlite3', database + '.rss.svg', options.merge("query-name" => 'rss'))
+  end
+end
+

data/lib/rbbt/workflow/accessor.rb

@@ -16,6 +16,10 @@ end
 
 module Workflow
 
+  def self.job_path?(path)
+    path.split("/")[-4] == "jobs"
+  end
+
   def log(status, message = nil, &block)
     Step.log(status, message, nil, &block)
   end
@@ -301,8 +305,9 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep.workflow = workflow
     dep.info[:name] = dep.name
-    dep.original_task_name ||= dep.task_name
+    dep.original_task_name ||= dep.task_name if dep.workflow
     begin
       workflow = Kernel.const_get workflow if String === workflow
       dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)

data/lib/rbbt/workflow/step/accessor.rb

@@ -94,17 +94,21 @@ class Step
       Log.debug "Saving job input #{name} (#{type}) into #{path}"
       case
       when Step === value
-        Open.link(value.path, path)
+        Open.ln_s(value.path, path)
       when type.to_s == "file"
         if String === value && File.exists?(value)
-          Open.link(value, path)
+          Open.ln_s(value, path)
         else
           Open.write(path + '.yaml', value.to_yaml)
         end
       when Array === value
-        Open.write(path, value * "\n")
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
       when IO === value
-        Open.write(path, value)
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
+        else
+          Open.write(path, value)
+        end
       else
         Open.write(path, value.to_s)
       end
@@ -117,13 +121,17 @@ class Step
     task_name = Symbol === job.overriden ? job.overriden : job.task_name
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
-    task_info = workflow.task_info(task_name)
-    input_types = task_info[:input_types]
-    task_inputs = task_info[:inputs]
-    input_defaults = task_info[:input_defaults]
+    if workflow
+      task_info = workflow.task_info(task_name)
+      input_types = task_info[:input_types]
+      task_inputs = task_info[:inputs]
+      input_defaults = task_info[:input_defaults]
+    else
+      task_info = input_types = task_inputs = input_defaults = {}
+    end
 
     inputs = {}
-    real_inputs = job.real_inputs || job.info[:real_inputs]
+    real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
       next unless real_inputs.include? name.to_sym
@@ -497,8 +505,8 @@ class Step
 
   def running? 
     return false if ! (started? || status == :ending)
-    pid = info[:pid]
-    return nil if pid.nil?
+    return nil unless Open.exist?(self.pid_file)
+    pid = Open.read(self.pid_file).to_i
 
     return false if done? or error? or aborted? 
 
@@ -522,8 +530,7 @@ class Step
   end
 
   def nopid?
-    pid = info[:pid] || Open.exists?(pid_file)
-    ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
+    ! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
   end
 
   def aborted?

data/lib/rbbt/workflow/step/dependencies.rb

@@ -103,7 +103,7 @@ class Step
       end
 
       job.dup_inputs unless status == 'done' or job.started?
-      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
+      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started? || ! Workflow.job_path?(job.path)
 
       canfail = ComputeDependency === job && job.canfail?
     end
@@ -130,7 +130,6 @@ class Step
     (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
   end
 
-
   def execute_dependency(dependency, log = true)
     task_name = self.task_name
     canfail_paths = self.canfail_paths

data/lib/rbbt/workflow/step/run.rb

@@ -122,7 +122,6 @@ class Step
       reject{|dependency| (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dependency) || Open.remote?(dependency.path) }.
       reject{|dependency| dependency.error? }.
       #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
-      #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
       select{|dependency| dependency.updatable? }.
       collect{|dependency| Workflow.relocate_dependency(self, dependency)}
   end
@@ -374,7 +373,6 @@ class Step
                 Log.exception $!
               ensure
                 Step.purge_stream_cache
-                set_info :pid, nil
                 Open.rm pid_file if Open.exist?(pid_file)
               end
             end
@@ -389,7 +387,6 @@ class Step
                 _clean_finished
               rescue
                 stop_dependencies
-                set_info :pid, nil
                 Open.rm pid_file if Open.exist?(pid_file)
               end
             end
@@ -450,7 +447,7 @@ class Step
     ensure 
       no_load = false unless IO === result
       Open.rm pid_file if Open.exist?(pid_file) unless no_load
-      set_info :pid, nil unless no_load
+      #set_info :pid, nil unless no_load
     end
   end
 
@@ -560,7 +557,7 @@ class Step
           RbbtSemaphore.post_semaphore(semaphore) if semaphore
           Kernel.exit! -1
         end
-        set_info :pid, nil
+        #set_info :pid, nil
       ensure
         RbbtSemaphore.post_semaphore(semaphore) if semaphore
       end

data/lib/rbbt/workflow/usage.rb

@@ -57,7 +57,7 @@ module Task
       puts Log.color(:magenta, "Input select options")
       puts
       selects.collect{|p| p}.uniq.each do |input,options|
-        puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| o.to_s} * ", ") << "\n"
+        puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
         puts unless Log.compact
       end
       puts

data/lib/rbbt/workflow/util/provenance.rb

@@ -22,11 +22,14 @@ class Step
   end
 
   def self.prov_report_msg(status, name, path, info = nil)
-    parts = path.sub(/\{.*/,'').sub(/#{Regexp.quote(name)}$/,'').split "/"
+    parts = path.sub(/\{.*/,'').split "/"
 
+    parts.pop
+    
     task = Log.color(:yellow, parts.pop)
     workflow = Log.color(:magenta, parts.pop)
-    if status.to_s == 'noinfo' and parts.last != 'jobs'
+    #if status.to_s == 'noinfo' && parts.last != 'jobs'
+    if ! Workflow.job_path?(path)
       task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
     end
 

data/share/rbbt_commands/slurm/clean

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean error or aborted jobs
+
+$ rbbt mnl [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-SBP--sbatch_parameters show sbatch parameters
+-dr--dry_run Do not erase anything
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+Log.severity = 4
+done, error, aborted, jobid, search, tail, sbatch_parameters, dry_run = options.values_at :done, :error, :aborted, :job, :search, :tail, :sbatch_parameters, :dry_run
+
+workdir = File.expand_path('~/rbbt-slurm')
+Path.setup(workdir)
+
+running_jobs = begin
+                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
+               end
+
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
+count = 0
+workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+ dir = File.dirname(fcmd)
+
+ if m = Open.read(fcmd).match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+  exe = m[1]
+ else
+  exe = nil
+ end
+
+ if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+
+ if File.exists?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'job.status'))
+  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exists?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+
+ aborted = error = true if aborted.nil? && error.nil?
+ if done || error || aborted || running || queued || jobid || search
+   select = false
+   select = true if done && exit_status && exit_status.to_i == 0 
+   select = true if error && exit_status && exit_status.to_i != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   select = select && jobid.split(",").include?(id) if jobid 
+   select = select && cmd.match(/#{search}/) if search
+   next unless select 
+ end
+
+
+ puts Log.color(:yellow, "**ERASING**")
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:sbatch_parameters]
+   puts Log.color(:magenta, "SBATCH parameters: ")
+   puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ end
+
+ if tail && File.exists?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
+
+ Open.rm_rf dir unless dry_run
+end
+
+puts 
+puts "Found #{count} jobs"
+