rbbt-util 5.31.10 → 5.31.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4909a13c518c1128adcdfc981153c04bb9f57c11b8501c65549b427fd0d647d5
-  data.tar.gz: 0b6f7089d1c78aec899dcfdec1784a0bbb56f3df7c84f24bedb6da0d79e058fc
+  metadata.gz: 9616d2b1893ed3444d432e99b4ef6ddaf7d61836ee1c23dbd4deb1efa767b80d
+  data.tar.gz: c5dc433a171c909eb2711ee37832fb094d271fcf6099f0394706af3cd8da1520
 SHA512:
-  metadata.gz: f041b99830f23f3bd8f6e29de8bec29ca5352d4a0873d36dc452b5c1d7a2d31a7d563247337980e8df06fbd9a1835ba04bab43bd9293d9484f82ab00fc534482
-  data.tar.gz: 2c80efb6433e60851c666cc7bb75a7e66a7a3e5813db3bd41c73c8511ef94a64b4ab8528da2a2abe104f2aad97d7ed69876931e5932c02a7ed2db1e5f470a3b1
+  metadata.gz: efe3b9c85ec947f900283ca4b2a3e651617558bfd7d85dbea80180ad377e18150df8fccde9efecdd2493d9e4f0ed7979dbdefe2f815e2480d910d5e507670422
+  data.tar.gz: 24bc0e0e6dda34cbf7ad4ea08cb1cbeb18b3bb55b42a79a4590183fd86e7247d7aeb648c31673ef5601f632828d71ae29594e3c6b8f5c7c860dab6592a373f68

data/lib/rbbt/hpc/batch.rb

@@ -6,6 +6,36 @@ module HPC
     end
   end
 
+  def self.batch_system(batch_system = 'auto')
+    case batch_system.to_s.downcase
+    when 'slurm'
+      HPC::SLURM
+    when 'lsf'
+      HPC::LSF
+    when 'auto'
+      case $previous_commands.last
+      when 'slurm'
+        HPC::SLURM
+      when 'lsf'
+        HPC::LSF
+      else
+        case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+        when 'slurm'
+          HPC::SLURM
+        when 'lsf'
+          HPC::LSF
+        else
+          case ENV["BATCH_SYSTEM"].to_s.downcase
+          when 'slurm'
+            HPC::SLURM
+          when 'lsf'
+            HPC::LSF
+          end
+        end
+      end
+    end
+  end
+
   module TemplateGeneration
     def exec_cmd(job, options = {})
       env_cmd     = Misc.process_options options, :env_cmd
@@ -30,12 +60,12 @@ module HPC
 -B "/.singularity_ruby_inline":"#{contain}/.singularity_ruby_inline":rw 
 -B "#{options[:batch_dir]}" \
 -B /scratch/tmp \
-#{ group != user_group ? "-B /gpfs/projects/#{user_group}" : "" } \
+          #{ group != user_group ? "-B /gpfs/projects/#{user_group}" : "" } \
 -B #{scratch_group_dir} \
 -B #{projects_group_dir} \
 -B /apps/ \
 -B ~/git:"#{contain}/git":ro \
-#{Open.exists?('~/.rbbt/software/opt/')? '-B ~/.rbbt/software/opt/:"/opt/":ro' : '' } \
+          #{Open.exists?('~/.rbbt/software/opt/')? '-B ~/.rbbt/software/opt/:"/opt/":ro' : '' } \
 -B ~/.rbbt:"#{contain}/home/":ro)
         end
 
@@ -74,7 +104,7 @@ module HPC
           [name, dep.path] * "="  
         end * ","
 
-        options[:override_deps] = override_deps
+        options[:override_deps] = override_deps unless override_deps.empty?
       end
 
       # Save inputs into inputs_dir
@@ -121,6 +151,9 @@ EOF
         :nodes,
         :queue,
         :singularity,
+        :singularity_img,
+        :singularity_opt_dir,
+        :singularity_ruby_inline,
         :sync,
         :task_cpus,
         :time,
@@ -163,6 +196,8 @@ EOF
       batch_options[:scratch_group_dir] = File.join('/gpfs/scratch/', group)
       batch_options[:projects_group_dir] = File.join('/gpfs/projects/', group)
 
+      batch_options[:singularity] = true if batch_options[:singularity_img]
+
       if batch_options[:contain_and_sync]
         if batch_options[:contain].nil?
           contain_base = Rbbt::Config.get(:contain_base_dir, :batch_contain, :batch, :default => "/scratch/tmp/rbbt-[USER]")
@@ -211,6 +246,7 @@ EOF
         :fexit  => File.join(batch_dir, 'exit.status'),
         :fsync  => File.join(batch_dir, 'sync.log'),
         :fsexit  => File.join(batch_dir, 'sync.status'),
+        :fenv  => File.join(batch_dir, 'env.vars'),
         :fcmd   => File.join(batch_dir, 'command.batch')
 
       batch_options
@@ -436,6 +472,7 @@ exit $exit_status
 
 # #{Log.color :green, "1. Prepare environment"}
 #{prepare_environment}
+env > #{batch_options[:fenv]}
 
 # #{Log.color :green, "2. Execute"}
 #{execute} 

data/lib/rbbt/hpc/orchestrate.rb

@@ -145,6 +145,8 @@ module HPC
       job_rules.delete :workflow
       
 
+      option_config_keys = options[:config_keys]
+
       job_options = IndiferentHash.setup(options.merge(job_rules).merge(:batch_dependencies => dep_ids))
       job_options.delete :orchestration_rules
 
@@ -154,6 +156,11 @@ module HPC
         job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
       end
 
+      if option_config_keys
+        option_config_keys = option_config_keys.gsub(/,\s+/,',') 
+        job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + option_config_keys : option_config_keys
+      end
+
       if options[:piggyback]
         manifest = options[:piggyback].uniq
         manifest += [job]
@@ -165,7 +172,7 @@ module HPC
         new_config_keys = self.job_rules(rules, job)[:config_keys]
         if new_config_keys
           new_config_keys = new_config_keys.gsub(/,\s+/,',') 
-          job_options[:config_keys] = job_options[:config_keys] ? new_config_keys + "," + job_options[:config_keys] : new_config_keys
+          job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + new_config_keys : new_config_keys
         end
 
         job_options.delete :piggyback

data/lib/rbbt/hpc/slurm.rb

@@ -8,9 +8,12 @@ module HPC
 
     def self.batch_system_variables
       <<-EOF
-let "MAX_MEMORY=$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" || let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
-BATCH_JOB_ID=$SLURM_JOB_ID
-BATCH_SYSTEM=SLURM
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( 1024 * $(nproc) / $SLURM_CPUS_PER_TASK )"
+[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" || MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+export MAX_MEMORY_DEFAULT
+export MAX_MEMORY
+export BATCH_JOB_ID=$SLURM_JOB_ID
+export BATCH_SYSTEM=SLURM
       EOF
     end
 
@@ -23,6 +26,7 @@ BATCH_SYSTEM=SLURM
       nodes          = Misc.process_options options, :nodes
       workdir        = Misc.process_options options, :workdir
       exclusive      = Misc.process_options options, :exclusive
+      highmem        = Misc.process_options options, :highmem
 
       batch_dir  = Misc.process_options options, :batch_dir
       batch_name = Misc.process_options options, :batch_name
@@ -45,6 +49,7 @@ BATCH_SYSTEM=SLURM
       EOF
 
       header << "#SBATCH --exclusive" << "\n" if exclusive
+      header << "#SBATCH --constraint=highmem" << "\n" if highmem
 
       header
     end

data/lib/rbbt/tsv/attach.rb

@@ -189,6 +189,7 @@ module TSV
     other.each do |k,v|
       self.zip_new k, v
     end
+    self
   end
 
   

data/lib/rbbt/util/cmd.rb

@@ -38,16 +38,17 @@ module CMD
         end
       end
       version_txt = ""
-      %w(--version -version --help).each do |f|
+      version = nil
+      ["--version", "-version", "--help", ""].each do |f|
         begin
-          version_txt += CMD.cmd("#{tool} #{f}").read
-          break
+          version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
+          version = Misc.scan_version_text(version_txt, tool)
+          break if version
         rescue
+          Log.exception $!
         end
       end
 
-      version = Misc.scan_version_text(version_txt, tool)
-
       @@init_cmd_tool[tool] = version || true
 
       return cmd if cmd
@@ -75,7 +76,8 @@ module CMD
     string = ""
     options.each do |option, value|
       raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=]+$/i
-      raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
+      #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
+      value = value.gsub("'","\\'") if value.to_s.include? "'"
 
       option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
 

data/lib/rbbt/util/misc/inspect.rb

@@ -431,8 +431,20 @@ module Misc
 
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
+    text.split("\n").each do |line|
+      next unless line =~ /\W#{cmd}\W/i
+      m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+      return m[1] if m
+      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
+      return m[1] if m
+      next unless line =~ /\Wversion\W/i
+      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+      return m[1] if m
+    end
     m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
-    return nil if m.nil?
-    m[1]
+    return m[1] if m
+    m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
+    return m[1] if m
+    nil
   end
 end

data/lib/rbbt/util/misc/options.rb

@@ -46,8 +46,8 @@ module Misc
   def self.hash2string(hash)
     hash.sort_by{|k,v| k.to_s}.collect{|k,v| 
       next unless %w(Symbol String Float Fixnum Integer Numeric TrueClass FalseClass Module Class Object).include? v.class.to_s
-      [ Symbol === k ? ":" << k.to_s : k,
-        Symbol === v ? ":" << v.to_s : v] * "="
+      [ Symbol === k ? ":" << k.to_s : k.to_s.chomp,
+        Symbol === v ? ":" << v.to_s : v.to_s.chomp] * "="
     }.compact * "#"
   end
 

data/lib/rbbt/util/misc/pipes.rb

@@ -35,10 +35,11 @@ module Misc
     res
   end
 
-  def self.with_fifo(path = nil, &block)
+  def self.with_fifo(path = nil, clean = true, &block)
     begin
       erase = path.nil?
       path = TmpFile.tmp_file if path.nil?
+      File.rm path if clean && File.exists?(path)
       File.mkfifo path
       yield path
     ensure

data/lib/rbbt/util/open.rb

@@ -11,7 +11,7 @@ module Open
   class OpenGzipError < StandardError; end
 
   REMOTE_CACHEDIR = File.join(ENV["HOME"], "/tmp/open_cache")
-  FileUtils.mkdir_p REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
+  #FileUtils.mkdir_p REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
 
   GREP_CMD = begin
                if ENV["GREP_CMD"] 
@@ -346,15 +346,18 @@ module Open
     target = File.join(target, File.basename(source)) if File.directory? target
     FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
     FileUtils.rm target if File.exist?(target)
+    FileUtils.rm target if File.symlink?(target)
     FileUtils.ln_s source, target
   end
 
   def self.ln(source, target, options = {})
     source = source.find if Path === source
     target = target.find if Path === target
+    source = File.realpath(source) if File.symlink?(source)
 
     FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
     FileUtils.rm target if File.exist?(target)
+    FileUtils.rm target if File.symlink?(target)
     FileUtils.ln source, target
   end
 

data/lib/rbbt/workflow/definition.rb

@@ -76,8 +76,10 @@ module Workflow
     extension workflow.tasks[oname].extension if workflow.tasks.include?(oname) unless @extension
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
-      dep = dependencies.last.join
+      dep = dependencies.last
+      dep.join
       raise dep.get_exception if dep.error?
+      raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
       set_info :result_type, dep.info[:result_type]
       forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
@@ -103,7 +105,11 @@ module Workflow
           Open.rm_rf self.files_dir 
           Open.link dep.files_dir, self.files_dir
         end
-        Open.link dep.path, self.path
+        if defined?(RemoteStep) && RemoteStep === dep
+          Open.write(self.tmp_path, Open.read(dep.path))
+        else
+          Open.link dep.path, self.path
+        end
       end
       nil
     end

data/lib/rbbt/workflow/examples.rb

@@ -54,7 +54,8 @@ module Workflow
             inputs[input.to_sym]  = YAML.load(Open.read(file))
           else
             if File.symlink?(file)
-              inputs[input.to_sym]  = File.readlink(file)
+              link_target = File.expand_path(File.readlink(file), File.dirname(file))
+              inputs[input.to_sym]  = link_target
             else
               inputs[input.to_sym]  = Open.realpath(file)
             end

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -20,7 +20,7 @@ class RemoteWorkflow
 
     def self.encode(url)
       begin
-        URI.encode(url)
+        URI::DEFAULT_PARSER.escape(url)
       rescue
         Log.warn $!.message
         url

data/lib/rbbt/workflow/step/accessor.rb

@@ -368,7 +368,7 @@ class Step
   end
 
   def self.log_progress(status, options = {}, path = nil, &block)
-    options = Misc.add_defaults options, :severity => Log::INFO, :file => path
+    options = Misc.add_defaults options, :severity => Log::INFO, :file => (@exec ? nil : path)
     max = Misc.process_options options, :max
     Log::ProgressBar.with_bar(max, options) do |bar|
       begin
@@ -394,7 +394,7 @@ class Step
     options = {} if options.nil?
 
     max = options[:max]
-    Log::ProgressBar.new_bar(max, {:desc => msg, :file => file(:progress)}.merge(options))
+    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
   end
 
   def self.log(status, message, path, &block)

data/lib/rbbt/workflow/usage.rb

@@ -215,7 +215,7 @@ module Workflow
 
       dep_tree = {[self, task_name] => dep_tree(task_name)}
       prov_tree = prov_tree(dep_tree)
-      if prov_tree && ! prov_tree.empty?
+      if prov_tree && ! prov_tree.empty? && prov_tree.split("\n").length > 2
 
         puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
         puts

data/lib/rbbt/workflow/util/trace.rb

@@ -1,12 +1,8 @@
 require 'rbbt/util/R'
 
 module Workflow
-  def self.trace(seed_jobs, options = {})
-
-    jobs = []
-    seed_jobs.each{|j| jobs << j; jobs += j.rec_dependencies}
-
-    data = TSV.setup({}, "Job~Workflow,Task,Start,End#:type=:list")
+  def self.trace_job_times(jobs, fix_gap = false)
+    data = TSV.setup({}, "Job~Code,Workflow,Task,Start,End#:type=:list")
     min_start = nil
     max_done = nil
     jobs.each do |job|
@@ -14,10 +10,10 @@ module Workflow
       started = job.info[:started]
       ddone = job.info[:done]
 
-      code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * "."
-      code = code + '.' + job.name
+      code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * " · "
+      code = job.name + " - " + code
 
-      data[code] = [job.workflow.to_s, job.task_name, started, ddone]
+      data[job.path] = [code,job.workflow.to_s, job.task_name, started, ddone]
       if min_start.nil?
         min_start = started
       else
@@ -39,7 +35,7 @@ module Workflow
       value["End"] - min_start
     end
 
-    if options[:fix_gap]
+    if fix_gap
       ranges = []
       data.through do |k,values|
         start, eend = values.values_at "Start.second", "End.second"
@@ -67,115 +63,155 @@ module Workflow
         gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
         value - gap
       end
+
+      total_gaps = Misc.sum(gaps.collect{|k,v| v})
+      Log.info "Total gaps: #{total_gaps} seconds"
     end
 
+    start = data.column("Start.second").values.flatten.collect{|v| v.to_f}.min
+    eend = data.column("End.second").values.flatten.collect{|v| v.to_f}.max
+    total = eend - start
+    Log.info "Total time elapsed: #{total} seconds"
+
+    data
+  end
+
+  def self.plot_trace_job_times(data, plot, width=800, height=800)
+    data.R <<-EOF, [:svg]
+rbbt.require('tidyverse')
+rbbt.require('ggplot2')
+
+names(data) <- make.names(names(data))
+data$id = data$Code
+data$content = data$Task
+data$start = data$Start
+data$end = data$End
+data$Project = data$Workflow
+
+tasks = data
+
+#theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
+theme_gantt <- function(base_size=11, base_family="Sans Serif") {
+  ret <- theme_bw(base_size, base_family) %+replace%
+    theme(panel.background = element_rect(fill="#ffffff", colour=NA),
+          axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
+          title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
+          panel.border = element_blank(), axis.line=element_blank(),
+          panel.grid.minor=element_blank(),
+          panel.grid.major.y = element_blank(),
+          panel.grid.major.x = element_line(size=0.5, colour="grey80"),
+          axis.ticks=element_blank(),
+          legend.position="bottom", 
+          axis.title=element_text(size=rel(1.2), family="Source Sans Pro Semibold"),
+          strip.text=element_text(size=rel(1.5), family="Source Sans Pro Semibold"),
+          strip.background=element_rect(fill="#ffffff", colour=NA),
+          panel.spacing.y=unit(1.5, "lines"),
+          legend.key = element_blank())
+
+  ret
+}
+
+tasks.long <- tasks %>%
+gather(date.type, task.date, -c(Code,Project, Task, id, Start.second, End.second)) %>%
+arrange(date.type, task.date) %>%
+mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
+
+x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
+
+timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) + 
+  geom_segment() + 
+  geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") + 
+  guides(colour=guide_legend(title=NULL)) +
+  labs(x=NULL, y=NULL) + 
+  theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
+
+rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, pointsize=6)
+    EOF
+  end
+
+  def self.trace_job_summary(jobs, report_keys = [])
     tasks_info = {}
 
+    report_keys = report_keys.collect{|k| k.to_s}
+
     jobs.each do |dep|
       next unless dep.info[:done]
       task = [dep.workflow, dep.task_name].compact.collect{|s| s.to_s} * "#"
-      info = tasks_info[task] ||= {}
+      info = tasks_info[task] ||= IndiferentHash.setup({})
+      dep_info = IndiferentHash.setup(dep.info)
 
-      time = dep.info[:done] - dep.info[:started]
+      time = dep_info[:done] - dep_info[:started]
       info[:time] ||= []
       info[:time] << time
 
-      cpus = nil
-      spark = false
-      shard = false
+      report_keys.each do |key|
+        info[key] = dep_info[key] 
+      end
+
       dep.info[:config_keys].select do |kinfo| 
         key, value, tokens = kinfo
-        key = key.to_s
-        cpus = value if key.include? 'cpu'
-        spark = value if key == 'spark'
-        shard = value if key == 'shard'
-      end
 
-      info[:cpus] = cpus || 1
-      info[:spark] = spark
-      info[:shard] = shard
+        info[key.to_s] = value if report_keys.include? key.to_s
+      end
     end
 
-    stats = TSV.setup({}, "Task~Calls,Avg. Time,Total Time,Cpus,Spark,Shard#:type=:list")
+    summary = TSV.setup({}, "Task~Calls,Avg. Time,Total Time#:type=:list")
 
     tasks_info.each do |task, info|
-      time_lists, cpus, spark, shard = info.values_at :time, :cpus, :spark, :shard
-      avg_time = Misc.mean(time_lists)
-      total_time = Misc.sum(time_lists)
+      time_lists = info[:time]
+      avg_time = Misc.mean(time_lists).to_i
+      total_time = Misc.sum(time_lists).to_i
       calls = time_lists.length
-      stats[task] = [calls, avg_time, total_time, cpus, spark, shard]
+      summary[task] = [calls, avg_time, total_time]
     end
 
-    raise "No jobs to process" if data.size == 0
+    report_keys.each do |key|
+      summary.add_field Misc.humanize(key) do |task|
+        tasks_info[task][key]
+      end
+    end if Array === report_keys && report_keys.any?
 
-    start = data.column("Start.second").values.flatten.collect{|v| v.to_f}.min
-    eend = data.column("End.second").values.flatten.collect{|v| v.to_f}.max
-    total = eend - start
-    Log.info "Total time elapsed: #{total} seconds"
+    summary
+  end
 
-    if options[:fix_gap]
-      total_gaps = Misc.sum(gaps.collect{|k,v| v})
-      Log.info "Total gaps: #{total_gaps} seconds"
+  def self.trace(seed_jobs, options = {})
+    jobs = []
+    seed_jobs.each do |step|
+      jobs += step.rec_dependencies + [step]
+
+      step.info[:archived_info].each do |path,ainfo|
+        archived_step = Step.new path
+        class << archived_step
+          self
+        end.define_method :info do
+          ainfo
+        end
+        jobs << archived_step
+      end if step.info[:archived_info]
     end
 
-    plot, width, height = options.values_at :plot, :width, :height
-    if plot
-      data.R <<-EOF, [:svg]
-    rbbt.require('tidyverse')
-    rbbt.require('ggplot2')
-
-    names(data) <- make.names(names(data))
-    data$id = rownames(data)
-    data$content = data$Task
-    data$start = data$Start
-    data$end = data$End
-    data$Project = data$Workflow
-
-    tasks = data
-
-    #theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
-    theme_gantt <- function(base_size=11, base_family="Sans Serif") {
-      ret <- theme_bw(base_size, base_family) %+replace%
-        theme(panel.background = element_rect(fill="#ffffff", colour=NA),
-              axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
-              title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
-              panel.border = element_blank(), axis.line=element_blank(),
-              panel.grid.minor=element_blank(),
-              panel.grid.major.y = element_blank(),
-              panel.grid.major.x = element_line(size=0.5, colour="grey80"),
-              axis.ticks=element_blank(),
-              legend.position="bottom", 
-              axis.title=element_text(size=rel(1.2), family="Source Sans Pro Semibold"),
-              strip.text=element_text(size=rel(1.5), family="Source Sans Pro Semibold"),
-              strip.background=element_rect(fill="#ffffff", colour=NA),
-              panel.spacing.y=unit(1.5, "lines"),
-              legend.key = element_blank())
-
-      ret
-    }
-
-    tasks.long <- tasks %>%
-    gather(date.type, task.date, -c(Project, Task, id, Start.second, End.second)) %>%
-    arrange(date.type, task.date) %>%
-    mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
-
-    x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
-
-    timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) + 
-      geom_segment() + 
-      geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") + 
-      guides(colour=guide_legend(title=NULL)) +
-      labs(x=NULL, y=NULL) + 
-      theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
-
-    rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, pointsize=6)
-      EOF
-    end
+    jobs = jobs.uniq.sort_by{|job| t = job.info[:done]; t || Open.mtime(job.path) || 0 }
+
+    data = trace_job_times(jobs, options[:fix_gap])
+
+    report_keys = options[:report_keys] || ""
+    report_keys = report_keys.split(/,\s*/) if String === report_keys
+    summary = trace_job_summary(jobs, report_keys)
+
+    raise "No jobs to process" if data.size == 0
+
+    plot, size, width, height = options.values_at :plot, :width, :height
+
+    size = 800 if size.nil?
+    width = size * 2 if width.nil?
+    height = size  if height.nil?
+
+    plot_trace_job_times(data, plot, width, height) if plot
 
     if options[:plot_data]
       data
     else
-      stats
+      summary
     end
 
   end

data/share/rbbt_commands/hpc/clean

@@ -36,33 +36,7 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
-HPC::BATCH_MODULE = case batch_system.to_s.downcase
-               when 'slurm'
-                 HPC::SLURM
-               when 'lsf'
-                 HPC::LSF
-               when 'auto'
-                 case $previous_commands.last
-                 when 'slurm'
-                   HPC::SLURM
-                 when 'lsf'
-                   HPC::LSF
-                 else
-                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
-                   when 'slurm'
-                     HPC::SLURM
-                   when 'lsf'
-                     HPC::LSF
-                   else
-                     case ENV["BATCH_SYSTEM"].to_s.downcase
-                     when 'slurm'
-                       HPC::SLURM
-                     when 'lsf'
-                       HPC::LSF
-                     end
-                   end
-                 end
-               end
+HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 

data/share/rbbt_commands/hpc/list

@@ -40,33 +40,7 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
-HPC::BATCH_MODULE = case batch_system.to_s.downcase
-               when 'slurm'
-                 HPC::SLURM
-               when 'lsf'
-                 HPC::LSF
-               when 'auto'
-                 case $previous_commands.last
-                 when 'slurm'
-                   HPC::SLURM
-                 when 'lsf'
-                   HPC::LSF
-                 else
-                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
-                   when 'slurm'
-                     HPC::SLURM
-                   when 'lsf'
-                     HPC::LSF
-                   else
-                     case ENV["BATCH_SYSTEM"].to_s.downcase
-                     when 'slurm'
-                       HPC::SLURM
-                     when 'lsf'
-                       HPC::LSF
-                     end
-                   end
-                 end
-               end
+HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
@@ -108,7 +82,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   cmd = nil
  end
 
- if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
+ if m = command_txt.match(/^export BATCH_SYSTEM=(.*)/)
   job_batch_system = m[1].downcase
  else
   job_batch_system = nil
@@ -235,6 +209,8 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
    when 'lsf'
      text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+   else
+     text = ""
    end
    lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
    puts Log.color :yellow, lines * "\n"

data/share/rbbt_commands/hpc/orchestrate

@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
 -cj--clean_job Clean job
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -33,33 +34,7 @@ EOF
 batch_system = $slurm_options.delete :batch_system
 batch_system ||= 'auto'
 
-HPC::BATCH_MODULE = case batch_system.to_s.downcase
-               when 'slurm'
-                 HPC::SLURM
-               when 'lsf'
-                 HPC::LSF
-               when 'auto'
-                 case $previous_commands.last
-                 when 'slurm'
-                   HPC::SLURM
-                 when 'lsf'
-                   HPC::LSF
-                 else
-                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
-                   when 'slurm'
-                     HPC::SLURM
-                   when 'lsf'
-                     HPC::LSF
-                   else
-                     case ENV["BATCH_SYSTEM"].to_s.downcase
-                     when 'slurm'
-                       HPC::SLURM
-                     when 'lsf'
-                       HPC::LSF
-                     end
-                   end
-                 end
-               end
+HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 

data/share/rbbt_commands/hpc/tail

@@ -40,33 +40,7 @@ end
 batch_system = options.delete :batch_system
 batch_system ||= 'auto'
 
-HPC::BATCH_MODULE = case batch_system.to_s.downcase
-               when 'slurm'
-                 HPC::SLURM
-               when 'lsf'
-                 HPC::LSF
-               when 'auto'
-                 case $previous_commands.last
-                 when 'slurm'
-                   HPC::SLURM
-                 when 'lsf'
-                   HPC::LSF
-                 else
-                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
-                   when 'slurm'
-                     HPC::SLURM
-                   when 'lsf'
-                     HPC::LSF
-                   else
-                     case ENV["BATCH_SYSTEM"].to_s.downcase
-                     when 'slurm'
-                       HPC::SLURM
-                     when 'lsf'
-                       HPC::LSF
-                     end
-                   end
-                 end
-               end
+HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 

data/share/rbbt_commands/hpc/task

@@ -11,6 +11,7 @@ $slurm_options = SOPT.get <<EOF
 -cj--clean_job Clean job
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -32,33 +33,7 @@ EOF
 batch_system = $slurm_options.delete :batch_system
 batch_system ||= 'auto'
 
-HPC::BATCH_MODULE = case batch_system.to_s.downcase
-               when 'slurm'
-                 HPC::SLURM
-               when 'lsf'
-                 HPC::LSF
-               when 'auto'
-                 case $previous_commands.last
-                 when 'slurm'
-                   HPC::SLURM
-                 when 'lsf'
-                   HPC::LSF
-                 else
-                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
-                   when 'slurm'
-                     HPC::SLURM
-                   when 'lsf'
-                     HPC::LSF
-                   else
-                     case ENV["BATCH_SYSTEM"].to_s.downcase
-                     when 'slurm'
-                       HPC::SLURM
-                     when 'lsf'
-                       HPC::LSF
-                     end
-                   end
-                 end
-               end
+HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 

data/share/rbbt_commands/log

@@ -31,7 +31,9 @@ if ARGV.empty?
     puts Rbbt.etc.log_severity.find + ' does not exist'                                                                                                                                  
   end                                                                                                                                                                                      
 else
-  Open.write(Rbbt.etc.log_severity, ARGV[0].upcase)
+  level = ARGV[0]
+  level = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE)[level.to_i] if level =~ /^\d+$/
+  Open.write(Rbbt.etc.log_severity, level.upcase)
 end
 
 

data/share/rbbt_commands/workflow/trace

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'rbbt/workflow'
+require 'rbbt/workflow/util/trace'
 
 require 'rbbt-util'
 require 'fileutils'
@@ -16,30 +17,26 @@ require 'rbbt/util/R'
 $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
 options = SOPT.setup <<EOF
-Examine the provenance of a job result
+Examine the execution trace of a job or set of jobs
 
 $ rbbt workflow trace <job-result>
 
 -h--help Help
+-fg--fix_gap Remove execution gaps
+-rk--report_keys* Config keys and info fields to report
+-p--plot* Plot file
 -w--width* Image Width
 -h--height* Image Height
--p--plot Plot file
 -s--size* Image Size (Height and Width)
--fg--fix_gap Remove execution gaps
 -pd--plot_data Print plot data
 EOF
 
 SOPT.usage if options[:help]
 
+
 files = ARGV
 plot = options[:plot]
 
-width, height, size = options.values_at :width, :height, :size
-
-size = 800 if size.nil?
-width = size if width.nil?
-height = size if height.nil?
-
 def get_step(file)
   file = File.expand_path(file)
   file = file.sub(/\.(info|files)/,'')
@@ -47,191 +44,8 @@ def get_step(file)
 end
 
 jobs = []
-files.each do |file|
-  step = get_step file
-
-  jobs += step.rec_dependencies + [step]
-
-  step.info[:archived_info].each do |path,ainfo|
-    archived_step = Step.new path
-    class << archived_step
-      self
-    end.define_method :info do
-      ainfo
-    end
-    jobs << archived_step
-  end if step.info[:archived_info]
-end
-
-jobs = jobs.select{|job| job.info[:done]}.sort_by{|job| job.info[:started]}
-
-data = TSV.setup({}, "Job~Workflow,Task,Start,End#:type=:list")
-min_start = nil
-max_done = nil
-jobs.each do |job|
-  next unless job.info[:done]
-  started = job.info[:started]
-  ddone = job.info[:done]
-
-  code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * "."
-  code = code + '.' + job.name
-
-  data[code] = [job.workflow.to_s, job.task_name, started, ddone]
-  if min_start.nil?
-    min_start = started
-  else
-    min_start = started if started < min_start
-  end
-
-  if max_done.nil?
-    max_done = ddone
-  else
-    max_done = ddone if ddone > max_done
-  end
-end
-
-data.add_field "Start.second" do |k,value|
-  value["Start"] - min_start
-end
-
-data.add_field "End.second" do |k,value|
-  value["End"] - min_start
-end
-
-if options[:fix_gap]
-  ranges = []
-  data.through do |k,values|
-    start, eend = values.values_at "Start.second", "End.second"
-
-    ranges << (start..eend)
-  end
-
-  gaps = {}
-  last = nil
-  Misc.collapse_ranges(ranges).each do |range|
-    start = range.begin
-    eend = range.end
-    if last
-      gaps[last] = start - last
-    end
-    last = eend
-  end
-
-  data.process "End.second" do |value,k,values|
-    gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
-    value - gap
-  end
-
-  data.process "Start.second" do |value,k,values|
-    gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
-    value - gap
-  end
+jobs = files.collect do |file|
+  get_step file
 end
 
-tasks_info = {}
-
-jobs.each do |dep|
-  next unless dep.info[:done]
-  task = [dep.workflow, dep.task_name].compact.collect{|s| s.to_s} * "#"
-  info = tasks_info[task] ||= {}
-
-  time = dep.info[:done] - dep.info[:started]
-  info[:time] ||= []
-  info[:time] << time
-
-  cpus = nil
-  spark = false
-  shard = false
-  dep.info[:config_keys].select do |kinfo| 
-    key, value, tokens = kinfo
-    key = key.to_s
-    cpus = value if key.include? 'cpu'
-    spark = value if key == 'spark'
-    shard = value if key == 'shard'
-  end
-
-  info[:cpus] = cpus || 1
-  info[:spark] = spark
-  info[:shard] = shard
-end
-
-stats = TSV.setup({}, "Task~Calls,Avg. Time,Total Time,Cpus,Spark,Shard#:type=:list")
-
-tasks_info.each do |task, info|
-  time_lists, cpus, spark, shard = info.values_at :time, :cpus, :spark, :shard
-  avg_time = Misc.mean(time_lists).to_i
-  total_time = Misc.sum(time_lists)
-  calls = time_lists.length
-  stats[task] = [calls, avg_time, total_time, cpus, spark, shard]
-end
-
-raise "No jobs to process" if data.size == 0
-
-start = data.column("Start.second").values.flatten.collect{|v| v.to_i}.min
-eend = data.column("End.second").values.flatten.collect{|v| v.to_i}.max
-total = eend - start
-Log.info "Total time elapsed: #{total} seconds"
-
-if options[:fix_gap]
-  total_gaps = Misc.sum(gaps.collect{|k,v| v})
-  Log.info "Total gaps: #{total_gaps} seconds"
-end
-
-if options[:plot_data]
-  puts data.to_s
-else
-  puts stats.to_s
-end
-
-if plot
-  data.R <<-EOF, [:svg]
-    rbbt.require('tidyverse')
-    rbbt.require('ggplot2')
-
-    names(data) <- make.names(names(data))
-    data$id = rownames(data)
-    data$content = data$Task
-    data$start = data$Start
-    data$end = data$End
-    data$Project = data$Workflow
-
-    tasks = data
-
-    #theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
-    theme_gantt <- function(base_size=11, base_family="Sans Serif") {
-      ret <- theme_bw(base_size, base_family) %+replace%
-        theme(panel.background = element_rect(fill="#ffffff", colour=NA),
-              axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
-              title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
-              panel.border = element_blank(), axis.line=element_blank(),
-              panel.grid.minor=element_blank(),
-              panel.grid.major.y = element_blank(),
-              panel.grid.major.x = element_line(size=0.5, colour="grey80"),
-              axis.ticks=element_blank(),
-              legend.position="bottom", 
-              axis.title=element_text(size=rel(1.2), family="Source Sans Pro Semibold"),
-              strip.text=element_text(size=rel(1.5), family="Source Sans Pro Semibold"),
-              strip.background=element_rect(fill="#ffffff", colour=NA),
-              panel.spacing.y=unit(1.5, "lines"),
-              legend.key = element_blank())
-
-      ret
-    }
-
-    tasks.long <- tasks %>%
-    gather(date.type, task.date, -c(Project, Task, id, Start.second, End.second)) %>%
-    arrange(date.type, task.date) %>%
-    mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
-
-    x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
-
-    timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) + 
-      geom_segment() + 
-      geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") + 
-      guides(colour=guide_legend(title=NULL)) +
-      labs(x=NULL, y=NULL) + 
-      theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
-
-    rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, pointsize=6)
-  EOF
-end
+puts Workflow.trace(jobs, options)

data/share/rbbt_commands/workflow/write_info

@@ -46,7 +46,21 @@ pid = step.info[:pid]
 host = step.info[:pid_hostname]
 
 step.rec_dependencies.each do |dep|
-  dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
-rescue
-  Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
+  begin
+    dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
+  rescue
+    Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
+  end
 end if recursive
+
+if recursive && step.info[:archived_info]
+  ad = step.info[:archived_info]
+  ad.each do |d,info|
+    begin
+      info[key] = value if (force || ! info.include?(key)) && (!check_pid || info[:pid].to_s == pid and info[:pid_hostname] == host)
+    rescue
+      Log.warn "Could no set info #{key} for archived_dep #{info[:path]}: #{$!.message}"
+    end 
+  end
+  step.set_info :archived_info, ad
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.31.10
+  version: 5.31.15
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-26 00:00:00.000000000 Z
+date: 2021-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake