rbbt-util 5.33.5 → 5.33.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bea55b2e4dd5972cf0e9646e84db4ce6ffd0556bb54796927640dbe3a3ba6611
-  data.tar.gz: 281d52e1685badaba8427e5ffd2226e1a1ab61d66df30cc7cd88acb79ef5aa18
+  metadata.gz: aaa014ffdad4e098212cc3f7cefc4d04ac62a7f7fcbf17bbbab9415c6253d9ef
+  data.tar.gz: b8a2a233686ec2f229ed79bb19d87d8eb79ad945142305f7b61dcdb11f5f7912
 SHA512:
-  metadata.gz: 459b75076980833dc1be3bd60d9fc98707270c626ab837e76a6c09180aaa2bd3328b4dea8776cf09c8fca0358e38258a30a90d17cd615450a38398c53bacfef0
-  data.tar.gz: 1bb336e1a592b64fbb3e87453042e3f6f220b76c92f5e9ff7550aa25401fd3e36a823f31bddb0e8be6878fa7646b28a010ac320d6fd8e00066bd1ebf2e9f774f
+  metadata.gz: 564242bef1dd92c2f53a5fe6c967624de26d7ae7eeb4da1ef2a1f682db9cf6d4b0bf0f8e76b3a600afe6af7bb443df28a8fe50e570c3c56265319eb607d6c1fc
+  data.tar.gz: ac7c01fa7e8fccdff8dfc861b82f28889a68f77886c198ff55b09558b36857ec9c29a8bae979bc7d088f311ad81515676bc7a4cc30097fd6c4b7586e6838c0e3

data/lib/rbbt/hpc/batch.rb

@@ -544,7 +544,7 @@ env > #{batch_options[:fenv]}
         Misc.add_defaults options, 
           :batch_dir => batch_dir, 
           :inputs_dir => File.join(batch_dir, "inputs_dir"),
-          :workflows => workflows_to_load * ","
+          :workflows => workflows_to_load.uniq * ","
 
         options[:procpath_performance] ||= File.join(batch_dir, "procpath##{procpath.gsub(',', '#')}") if procpath
 

data/lib/rbbt/hpc/orchestrate/chains.rb

@@ -1,6 +1,7 @@
 module HPC
   module Orchestration
     def self.check_chains(chains, job)
+      return [] if Symbol === job.overriden
       matches = []
       chains.each do |name, chain|
         next unless chain[:tasks].include?(job.workflow.to_s)
@@ -36,95 +37,6 @@ module HPC
       (job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
     end
 
-    #def self.job_workload(job)
-    #  workload = []
-    #  heap = []
-    #  heap << job
-    #  while job = heap.pop
-    #    next if job.done?
-    #    workload << job
-    #    heap.concat job_dependencies(job)
-    #    heap.uniq!
-    #  end
-    #  workload.uniq
-    #end
-
-    #def self.top_level_job(jobs)
-    #  top = jobs.select do |job|
-    #    (jobs - job_workload(job)).empty? &&
-    #      (job_workload(job) - jobs).select{|j| (job_workload(j) & jobs).any? }.empty?
-    #  end
-    #  return nil if top.length != 1
-    #  top.first
-    #end
-
-    #def self.job_chains(rules, job)
-    #  workload = job_workload(job)
-    #  chains = parse_chains(rules)
-
-    #  chain_jobs = {}
-    #  workload.each do |job|
-    #    check_chains(chains, job).each do |match|
-    #      chain_jobs[match] ||= []
-    #      chain_jobs[match] << job
-    #    end
-    #  end
-
-    #  job_chains = []
-
-    #  seen = []
-    #  chain_jobs.sort_by{|name,jobs| jobs.length }.reverse.each do |name,jobs|
-    #    remain = jobs - seen
-    #    next unless remain.length > 1
-    #    top_level_job = top_level_job(jobs)
-    #    next if top_level_job.nil?
-    #    job_chains << {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
-    #    seen.concat remain
-    #  end
-
-    #  job_chains
-    #end
-
-    #def self._job_chains(rules, job)
-    #  workload = job_workload(job)
-    #  chains = parse_chains(rules)
-
-    #  matches = check_chains(chains, job)
-
-    #  job_chains = {}
-    #  job.dependencies.each do |dep|
-    #    dep_chains = _job_chains(rules, dep)
-    #    matches.each do |match|
-    #      if dep_chains[match] && dep_chains[match].include?(dep)
-    #        dep_chains[match].prepend job
-    #      end
-    #    end
-    #    job_chains.merge!(dep_chains)
-    #  end
-
-    #  matches.each do |match|
-    #    job_chains[match] ||= [job]
-    #  end
-
-    #  job_chains
-    #end
-
-    #def self.job_chains(rules, job)
-    #  job_chains = self._job_chains(rules, job)
-    #  iif job_chains
-    #  chains = parse_chains(rules)
-
-    #  seen = []
-    #  job_chains.collect do |name,jobs|
-    #    remain = jobs - seen
-    #    next unless remain.length > 1
-    #    top_level_job = top_level_job(jobs)
-    #    next if top_level_job.nil?
-    #    seen.concat remain
-    #    {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
-    #  end.compact
-    #end
-
     def self.job_chains(rules, job)
       chains = self.parse_chains(rules)
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -5,6 +5,23 @@ require 'rbbt/hpc/orchestrate/batches'
 module HPC
   module Orchestration
 
+    def prepare_for_execution(job)
+      rec_dependencies = job.rec_dependencies(true)
+
+      return if rec_dependencies.empty?
+
+      all_deps = rec_dependencies + [job]
+
+      all_deps.each do |dep|
+        begin
+          Step.prepare_for_execution(dep)
+        rescue RbbtException
+          next
+        end
+      end
+
+    end
+
     def orchestrate_job(job, options)
       options.delete "recursive_clean"
       options.delete "clean_task"
@@ -14,6 +31,9 @@ module HPC
       options.delete "detach"
       options.delete "jobname"
 
+      Log.high "Prepare for exec"
+      prepare_for_execution(job)
+
       if options[:orchestration_rules]
         rules = YAML.load(Open.read(options[:orchestration_rules]))
       elsif Rbbt.etc.slurm["default.yaml"].exists?
@@ -25,6 +45,7 @@ module HPC
       IndiferentHash.setup(rules)
 
       batches = HPC::Orchestration.job_batches(rules, job)
+      Log.high "Compute #{batches.length} batches"
 
       batch_ids = {}
       while batches.any?
@@ -60,7 +81,9 @@ module HPC
 
         if options[:dry_run]
           puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-          puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          puts Log.color(:magenta, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          puts Log.color(:yellow, "Path: ") + top[:top_level].path
+          puts Log.color(:yellow, "Options: ") + job_options.inspect
           batch_ids[top] = top[:top_level].task_signature
         else
           id = run_job(top[:top_level], job_options)

data/lib/rbbt/resource/path.rb

@@ -167,8 +167,7 @@ module Path
 
     @path ||= {}
     rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil 
-    key_elems = [where, caller_lib, rsearch_paths, paths]
-    key = Misc.obj2digest(key_elems.inspect)
+    key = [where, caller_lib, rsearch_paths, paths].inspect
     self.sub!('~/', Etc.getpwuid.dir + '/') if self.include? "~"
 
     return @path[key] if @path[key]

data/lib/rbbt/resource/util.rb

@@ -1,13 +1,26 @@
 module Path
 
   def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
-    file = caller.reject{|l| 
-      l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
-      l =~ /rbbt\/resource\/path\.rb/ or
-      l =~ /rbbt\/persist.rb/ or
-      l =~ /rbbt\/util\/misc\.rb/ or
-      l =~ /progress-monitor\.rb/ 
-    }.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
+    #file = caller.reject{|l| 
+    #  l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+    #  l =~ /rbbt\/resource\/path\.rb/ or
+    #  l =~ /rbbt\/persist.rb/ or
+    #  l =~ /rbbt\/util\/misc\.rb/ or
+    #  l =~ /progress-monitor\.rb/ 
+    #}.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
+    
+    
+    if file.nil?
+      caller_dup = caller.dup
+      while file = caller_dup.shift
+        break unless file =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+          file =~ /rbbt\/resource\/path\.rb/ or
+          file =~ /rbbt\/persist.rb/ or
+          file =~ /rbbt\/util\/misc\.rb/ or
+          file =~ /progress-monitor\.rb/ 
+      end
+      file = file.sub(/\.rb[^\w].*/,'.rb')
+    end
 
     relative_to = [relative_to] unless Array === relative_to
     file = File.expand_path(file)

data/lib/rbbt/util/migrate.rb

@@ -1,4 +1,6 @@
 module Rbbt 
+
+  prepare_for_execution(job)
   def self.migrate_source_paths(path, resource = Rbbt, source = nil)
     if source
       lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
@@ -44,6 +46,8 @@ puts resource[path].find(search_path)
     excludes += (options[:exclude] || "").split(/,\s*/)
     excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
 
+    hard_link = options[:hard_link]
+
     other = options[:other] || []
 
     test_str = options[:test] ? '-nv' : ''
@@ -82,6 +86,8 @@ puts resource[path].find(search_path)
         # rsync_args = "-avztAXHP --copy-unsafe-links"
         rsync_args = "-avztAHP --copy-unsafe-links"
 
+        rsync_args << " --link-dest '#{source_path}'" if hard_link && ! options[:source]
+
         cmd = "rsync #{rsync_args} #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
 
         cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]

data/lib/rbbt/util/misc/inspect.rb

@@ -80,7 +80,7 @@ module Misc
       end
     when (defined? TSV and TSV)
       obj.with_unnamed do
-        "TSV:{"<< fingerprint(obj.all_fields|| []).inspect << ";" << fingerprint(obj.keys).inspect << "}"
+        "TSV:{"<< fingerprint(obj.all_fields|| []) << ";" << fingerprint(obj.keys) << "}"
       end
     when Hash
       if obj.length > 10
@@ -297,14 +297,14 @@ module Misc
     str = case obj
           when nil
             'nil'
+          when Symbol 
+            obj.to_s
           when TrueClass
             'true'
           when FalseClass
             'false'
           when Hash
             "{"<< obj.collect{|k,v| obj2str(k) + '=>' << obj2str(v)}*"," << "}"
-          when Symbol 
-            obj.to_s
           when (defined?(Path) and Path)
             if defined?(Step) && Open.exists?(Step.info_file(obj))
               obj2str(Workflow.load_step(obj))
@@ -324,9 +324,10 @@ module Misc
             end
           when String
             good_filename = Misc.is_filename?(obj, false) && ! %w(. ..).include?(obj) && %w(. /).include?(obj[0])
-            is_path = Path === obj
-            if good_filename && ! is_path
-              obj2str Path.setup(obj.dup)
+            if good_filename 
+              obj = obj.dup
+              obj.extend Path
+              obj2str obj
             else
               obj = obj.chomp if String === obj
               if obj.length > HASH2MD5_MAX_STRING_LENGTH
@@ -339,7 +340,7 @@ module Misc
             if obj.length > HASH2MD5_MAX_ARRAY_LENGTH
               "[" << sample_large_obj(obj, HASH2MD5_MAX_ARRAY_LENGTH).collect{|v| obj2str(v)} * "," << "]"
             else
-              "[" << obj.collect{|v| obj2str(v)} * "," << "]"
+              "[" << obj.collect{|v| obj2str(v) } * "," << "]"
             end
           when TSV::Parser
             remove_long_items(obj)

data/lib/rbbt/workflow/step/run.rb

@@ -181,8 +181,8 @@ class Step
     #  end
     #end
 
-    Log.high "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
-    Log.high "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
+    Log.medium "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
+    Log.medium "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
 
     outdated_time + outdated_dep
   end

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -5,10 +5,10 @@ module Workflow
 
     def self.job_workload(job)
       workload = {job => []}
-      return workload if job.done?
+      return workload if job.done? && ! job.dirty?
 
       job.dependencies.each do |dep|
-        next if dep.done?
+        next if dep.done? && ! job.dirty?
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep
@@ -16,7 +16,7 @@ module Workflow
       end
 
       job.input_dependencies.each do |dep|
-        next if dep.done?
+        next if dep.done? && ! job.dirty?
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep

data/share/Rlib/plot.R

@@ -38,3 +38,4 @@ geom_entity <- function (real.geom = NULL, mapping = NULL, data = NULL, stat = "
     rg
 }
 
+rbbt.ggplot2.rotate_x_labels <- function(){ theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) }

data/share/install/software/lib/install_helpers

@@ -185,6 +185,10 @@ compile(){
     local name=$1; shift
     local extra="$@"
 
+    if [ -f bootstrap ]; then
+      ./bootstrap
+    fi
+
     if [ -f Makefile -o -f makefile ]; then
         make -j 4 || exit -1
         make install || echo "No install"

data/share/rbbt_commands/hpc/list

@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
 -l--long Show more entries
+-c--compressed Show compressed information about entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
+ count += 1
+
+ if options[:compressed] 
+   status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
+   if different_system
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
+   else
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+   end 
+   prog_rep = []
+   if options[:progress] 
+     step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+     if step_line
+       require 'rbbt/workflow'
+       step_path = step_line.split(": ").last.strip
+       step = Step.new step_path
+       step.load_dependencies_from_info
+       has_bar = false
+       (step.rec_dependencies + [step]).reverse.each do |j|
+         next if j.done? 
+         if j.file(:progress).exists?
+           bar = Log::ProgressBar.new 
+           bar.load(j.file(:progress).yaml)
+           prog_rep << bar.report_msg.split("·")[0..1]
+         end
+       end
+     end
+   end
+   workflow, task, name = step_path.split("/")[-3..-1]
+   job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
+   puts [job_str, status, prog_rep ].flatten * " "
+   next
+ end
+
  puts Log.color :blue, dir
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    cpu_average = {}
    rss_average = {}
    perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
-    time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
-    time = time.to_f
-
-    cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
-    cpu_average[stat_pid] ||= {}
-    cpu_average[stat_pid][time] ||= []
-    cpu_average[stat_pid][time] << cpu.to_f
-    rss_average[time] ||= []
-    rss_average[time] << rss.to_f * page_size
+     time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
+     time = time.to_f
+
+     cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
+     cpu_average[stat_pid] ||= {}
+     cpu_average[stat_pid][time] ||= []
+     cpu_average[stat_pid][time] << cpu.to_f
+     rss_average[time] ||= []
+     rss_average[time] << rss.to_f * page_size
    end
 
    ticks = 0
    cpu_average.each do |stat_pid, cpu_average_pid|
-    start = cpu_average_pid.keys.sort.first
-    eend = cpu_average_pid.keys.sort.last
-    ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
+     start = cpu_average_pid.keys.sort.first
+     eend = cpu_average_pid.keys.sort.last
+     ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
    end
    start = rss_average.keys.sort.first
    eend = rss_average.keys.sort.last
@@ -284,7 +319,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
    end
  end
 
@@ -309,8 +344,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    end
  end
 
- count += 1
-
 end
 
 puts 

data/share/rbbt_commands/lsf/list

@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
 -l--long Show more entries
+-c--compressed Show compressed information about entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
+ count += 1
+
+ if options[:compressed] 
+   status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
+   if different_system
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
+   else
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+   end 
+   prog_rep = []
+   if options[:progress] 
+     step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+     if step_line
+       require 'rbbt/workflow'
+       step_path = step_line.split(": ").last.strip
+       step = Step.new step_path
+       step.load_dependencies_from_info
+       has_bar = false
+       (step.rec_dependencies + [step]).reverse.each do |j|
+         next if j.done? 
+         if j.file(:progress).exists?
+           bar = Log::ProgressBar.new 
+           bar.load(j.file(:progress).yaml)
+           prog_rep << bar.report_msg.split("·")[0..1]
+         end
+       end
+     end
+   end
+   workflow, task, name = step_path.split("/")[-3..-1]
+   job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
+   puts [job_str, status, prog_rep ].flatten * " "
+   next
+ end
+
  puts Log.color :blue, dir
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    cpu_average = {}
    rss_average = {}
    perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
-    time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
-    time = time.to_f
-
-    cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
-    cpu_average[stat_pid] ||= {}
-    cpu_average[stat_pid][time] ||= []
-    cpu_average[stat_pid][time] << cpu.to_f
-    rss_average[time] ||= []
-    rss_average[time] << rss.to_f * page_size
+     time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
+     time = time.to_f
+
+     cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
+     cpu_average[stat_pid] ||= {}
+     cpu_average[stat_pid][time] ||= []
+     cpu_average[stat_pid][time] << cpu.to_f
+     rss_average[time] ||= []
+     rss_average[time] << rss.to_f * page_size
    end
 
    ticks = 0
    cpu_average.each do |stat_pid, cpu_average_pid|
-    start = cpu_average_pid.keys.sort.first
-    eend = cpu_average_pid.keys.sort.last
-    ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
+     start = cpu_average_pid.keys.sort.first
+     eend = cpu_average_pid.keys.sort.last
+     ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
    end
    start = rss_average.keys.sort.first
    eend = rss_average.keys.sort.last
@@ -284,7 +319,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
    end
  end
 
@@ -309,8 +344,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    end
  end
 
- count += 1
-
 end
 
 puts 

data/share/rbbt_commands/slurm/list

@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
 -l--long Show more entries
+-c--compressed Show compressed information about entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
+ count += 1
+
+ if options[:compressed] 
+   status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
+   if different_system
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
+   else
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+   end 
+   prog_rep = []
+   if options[:progress] 
+     step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+     if step_line
+       require 'rbbt/workflow'
+       step_path = step_line.split(": ").last.strip
+       step = Step.new step_path
+       step.load_dependencies_from_info
+       has_bar = false
+       (step.rec_dependencies + [step]).reverse.each do |j|
+         next if j.done? 
+         if j.file(:progress).exists?
+           bar = Log::ProgressBar.new 
+           bar.load(j.file(:progress).yaml)
+           prog_rep << bar.report_msg.split("·")[0..1]
+         end
+       end
+     end
+   end
+   workflow, task, name = step_path.split("/")[-3..-1]
+   job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
+   puts [job_str, status, prog_rep ].flatten * " "
+   next
+ end
+
  puts Log.color :blue, dir
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    cpu_average = {}
    rss_average = {}
    perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
-    time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
-    time = time.to_f
-
-    cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
-    cpu_average[stat_pid] ||= {}
-    cpu_average[stat_pid][time] ||= []
-    cpu_average[stat_pid][time] << cpu.to_f
-    rss_average[time] ||= []
-    rss_average[time] << rss.to_f * page_size
+     time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
+     time = time.to_f
+
+     cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
+     cpu_average[stat_pid] ||= {}
+     cpu_average[stat_pid][time] ||= []
+     cpu_average[stat_pid][time] << cpu.to_f
+     rss_average[time] ||= []
+     rss_average[time] << rss.to_f * page_size
    end
 
    ticks = 0
    cpu_average.each do |stat_pid, cpu_average_pid|
-    start = cpu_average_pid.keys.sort.first
-    eend = cpu_average_pid.keys.sort.last
-    ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
+     start = cpu_average_pid.keys.sort.first
+     eend = cpu_average_pid.keys.sort.last
+     ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
    end
    start = rss_average.keys.sort.first
    eend = rss_average.keys.sort.last
@@ -284,7 +319,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
    end
  end
 
@@ -309,8 +344,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    end
  end
 
- count += 1
-
 end
 
 puts 

data/share/rbbt_commands/workflow/prov

@@ -37,7 +37,7 @@ file = ARGV.shift
 $seen = []
 def get_step(file)
   file = File.expand_path(file)
-  file = file.sub(/\.(info|files)/,'')
+  file = file.sub(/\.(info|files)$/,'')
   $seen << file
   Workflow.load_step file
 end

data/share/rbbt_commands/workflow/task

@@ -219,8 +219,8 @@ help = !!options.delete(:help)
 do_fork = !!options.delete(:fork)
 detach = !!options.delete(:detach)
 do_exec = !!options.delete(:exec)
-clean = !!options.delete(:clean)
 clean_task = options.delete(:clean_task)
+clean = !!options.delete(:clean) || clean_task
 override_deps = options.delete(:override_deps)
 recursive_clean = !!options.delete(:recursive_clean)
 out = options.include?(:output) ? File.open(options[:output], 'wb') : STDOUT

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.5
+  version: 5.33.8
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-04 00:00:00.000000000 Z
+date: 2022-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake