rbbt-util 5.30.6 → 5.30.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a9ed1ed74562cdb81e14bb2b2213398646f76e532bdee48e67f9ff2f5330aad
-  data.tar.gz: e54ecf83a0b4323ffe98f8968f4a65acc2aae2ee5437330f44f14dc17c450ec5
+  metadata.gz: '009c7b1e63d04078314d77556ca0a68c9a29d875a5cd69ff57335650bd20823b'
+  data.tar.gz: 9a6fb144f833fff84a234b226a076e24355ab3733500b09c1a96bb1d80d004f8
 SHA512:
-  metadata.gz: 4d2eb887a184e24d78e8c565cefc25f09246d14e0f22d9a9037447806489cb4148fb2d9d3aad4cfe9bca1f256c0ec43afcc343047f20f974e6facea3214f40c0
-  data.tar.gz: 7f593e22911a501e353b6eb49830291a29b038474fd7bfc182e402f255219fc1eaf07311ad334118b43c275abc9ab2a5198d68ccbcdce7f3b4636c38dc698afd
+  metadata.gz: 58bd3482675c9e55b71fe1b6fa79dec4f23ac40d321067c19412c762d8de30983918ab9866869cb2a93c590e1f2a5f7c4228c4ba54d1c4e247a627d5948707db
+  data.tar.gz: 0f7f3c371fd6dcacf0050fcaf5e8ea3da81e6665b7640bbc596a718a386a7fd563e89bd35189a8ccb3c45e0badcf52bc37aa585b18753a31ae5178c51c0dcd3c

data/lib/rbbt/hpc/orchestrate.rb

@@ -8,7 +8,7 @@ module HPC
       task_name = job.overriden.to_s if Symbol === job.overriden
 
       defaults = rules["defaults"] || {}
-      defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
+      defaults = defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
 
       job_rules = IndiferentHash.setup(defaults.dup)
 
@@ -53,79 +53,159 @@ module HPC
       job_rules
     end
 
-    def self.get_job_dependencies(job, job_rules)
+    def self.get_job_dependencies(job, job_rules = nil)
       deps = job.dependencies || []
       deps += job.input_dependencies || []
       deps
     end
 
-    def self.orchestrate_job(job, options, skip = false, seen = {}, chains = {})
-      return if job.done?
-      return unless job.path.split("/")[-4] == "jobs"
-      seen[:orchestration_target_job] ||= job
-
-      options.delete "recursive_clean"
-      options.delete "clean_task"
-      options.delete "clean"
-      options.delete "tail"
-      options.delete "printfile"
-      options.delete "detach"
+    def self.get_recursive_job_dependencies(job)
+      deps = get_job_dependencies(job) 
+      (deps + deps.collect{|dep| get_recursive_job_dependencies(dep) }).flatten
+    end
 
-      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
-      rules ||= {}
-      IndiferentHash.setup(rules)
+    def self.piggyback(job, job_rules, job_deps)
+      return false unless job_rules["skip"]
+      final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
+      final_deps = final_deps.reject{|dep| dep.done? }
+      return final_deps.first if final_deps.length == 1
+      return false
+    end
 
+    def self.get_chains(job, rules, chains = {})
       job_rules = self.job_rules(rules, job)
+      job_deps = get_job_dependencies(job)
+
+      input_deps = []
+      job.rec_dependencies.each do |dep|
+        input_deps.concat dep.input_dependencies
+      end
 
-      deps = get_job_dependencies(job, job_rules)
+      job_deps.each do |dep|
+        input_deps.concat dep.input_dependencies
+        get_chains(dep, rules, chains)
+      end
 
-      chains[job.path] ||= []
-      dep_ids = deps.collect do |dep|
-        skip_dep = job_rules["chain_tasks"] &&
+      job_deps.select do |dep|
+        chained = job_rules["chain_tasks"] &&
           job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
           job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
 
-          chains[job.path] << dep if skip_dep
+        dep_skip = dep.done? && ! input_deps.include?(dep) && self.job_rules(rules, dep)["skip"] 
+        chained || dep_skip
+      end.each do |dep|
+        chains[job] ||= [] 
+        chains[job] << dep 
+        chains[job].concat chains[dep] if chains[dep]
+      end
+
+      chains
+    end
+
+    def self.workload(job, rules, chains, options, seen = nil)
+      return [] if job.done?
+      if seen.nil?
+        seen = {}
+        target_job = true
+      end
 
-        deps = seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen, chains)
-        if job.canfail_paths.include? dep.path
-          [deps].flatten.compact.collect{|id| ['canfail', id] * ":"}
+      job_rules = self.job_rules(rules, job)
+      job_deps = get_job_dependencies(job)
+
+      chain = chains[job]
+      chain = chain.reject{|j| seen.include? j.path} if chain
+      chain = chain.reject{|dep| dep.done? } if chain
+      piggyback = piggyback(job, job_rules, job_deps)
+      dep_ids = job_deps.collect do |dep|
+        seen[dep.path] ||= nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
+        next_options = IndiferentHash.setup(options.dup)
+        if piggyback and piggyback == dep
+          next_options[:piggyback] ||= []
+          next_options[:piggyback].push job
+          ids = workload(dep, rules, chains, next_options, seen)
         else
-          deps
+          next_options.delete :piggyback
+          ids = workload(dep, rules, chains, next_options, seen)
         end
-      end.flatten.compact.uniq
 
-      skip = true if job_rules[:skip]
-      return dep_ids if skip and seen[:orchestration_target_job] != job
+        ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
+
+        seen[dep.path] = ids
+        ids
+      end.compact.flatten.uniq
+
+      return seen[job.path] || dep_ids if seen.include?(job.path)
+
+      if piggyback and seen[piggyback.path]
+        return seen[job.path] = seen[piggyback.path] 
+      end
 
       job_rules.delete :chain_tasks
       job_rules.delete :tasks
       job_rules.delete :workflow
       
-      config_keys = job_rules.delete(:config_keys)
 
       job_options = IndiferentHash.setup(options.merge(job_rules).merge(:slurm_dependencies => dep_ids))
       job_options.delete :orchestration_rules
+
+      config_keys = job_rules.delete(:config_keys)
       if config_keys
         config_keys.gsub!(/,\s+/,',') 
         job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
       end
 
-      manifest = []
-      stack = [job]
-      while dep = stack.pop
-        manifest << dep
-        stack += chains[dep.path] if chains[dep.path]
+      if options[:piggyback]
+        manifest = options[:piggyback].uniq
+        manifest += [job]
+        manifest.concat chain if chain
+
+        job = options[:piggyback].first
+
+        job_rules = self.job_rules(rules, job)
+        new_config_keys = self.job_rules(rules, job)[:config_keys]
+        if new_config_keys
+          new_config_keys = new_config_keys.gsub(/,\s+/,',') 
+          job_options[:config_keys] = job_options[:config_keys] ? new_config_keys + "," + job_options[:config_keys] : new_config_keys
+        end
+
+        job_options.delete :piggyback
+      else
+        manifest = [job]
+        manifest.concat chain if chain
       end
 
-      job_options[:manifest] = manifest.uniq.collect{|dep| dep.workflow_short_path}
+      manifest.uniq!
+
+      job_options[:manifest] = manifest.collect{|j| j.task_signature }
+
+      job_options[:config_keys] = job_options[:config_keys].split(",").uniq * "," if job_options[:config_keys]
 
       if options[:dry_run]
-        puts Log.color(:yellow, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-        []
+        puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
+        puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:slurm_dependencies]*", ")
+        job_options[:manifest].first
       else
         run_job(job, job_options)
       end
     end
+
+
+    def self.orchestrate_job(job, options)
+      options.delete "recursive_clean"
+      options.delete "clean_task"
+      options.delete "clean"
+      options.delete "tail"
+      options.delete "printfile"
+      options.delete "detach"
+
+      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
+      rules ||= {}
+      IndiferentHash.setup(rules)
+
+      chains = get_chains(job, rules)
+
+      workload(job, rules, chains, options)
+    end
+
   end
 end

data/lib/rbbt/hpc/slurm.rb

@@ -21,6 +21,8 @@ module HPC
       exclusive        = options.delete :exclusive
       highmem          = options.delete :highmem
 
+      slurm_step_path  = options.delete :slurm_step_path
+
       manifest         = options.delete :manifest
 
       queue            = options.delete(:queue) || Rbbt::Config.get('queue', :slurm_queue, :slurm, :SLURM, :default => 'bsc_ls')
@@ -264,6 +266,10 @@ EOF
 #MANIFEST: #{manifest * ", "}
       EOF
 
+      header +=<<-EOF if slurm_step_path
+#STEP_PATH: #{slurm_step_path}
+      EOF
+
       header +=<<-EOF
 #CMD: #{rbbt_cmd}
       EOF
@@ -536,7 +542,9 @@ EOF
       dependencies     = options.delete :slurm_dependencies
       procpath         = options.delete :SLURM_procpath
 
-      options[:jobname] = job.clean_name
+      options[:jobname]   = job.clean_name
+      options[:slurm_step_path] = job.path
+
       log_level         = options.delete :log
       log_level       ||= Log.severity
 

data/lib/rbbt/tsv/manipulate.rb

@@ -717,7 +717,7 @@ module TSV
   def transpose(key_field = "Unkown ID")
     case type
     when :single, :flat
-      transpose_list self.to_list, key_field
+      self.to_list.transpose_list key_field
     when :list
       transpose_list key_field
     when :double

data/lib/rbbt/util/cmd.rb

@@ -101,6 +101,7 @@ module CMD
     no_fail    = options.delete(:nofail) if no_fail.nil?
     no_wait    = options.delete(:no_wait)
     xvfb       = options.delete(:xvfb)
+    bar        = options.delete(:progress_bar)
 
     dont_close_in  = options.delete(:dont_close_in)
 
@@ -183,6 +184,7 @@ module CMD
 
       err_thread = Thread.new do
         while line = serr.gets
+          bar.process(line) if bar
           sout.log = line
           Log.log "STDERR [#{pid}]: " +  line, stderr 
         end if Integer === stderr and log
@@ -220,6 +222,8 @@ module CMD
   def self.cmd_pid(*args)
     all_args = *args
 
+    bar = all_args.last[:progress_bar] if Hash === all_args.last
+
     all_args << {} unless Hash === all_args.last
 
     level = all_args.last[:log] || 0
@@ -233,17 +237,27 @@ module CMD
     io = cmd(*all_args)
     pid = io.pids.first
 
+    line = "" if bar
     while c = io.getc
       STDERR << c if Log.severity <= level
+      line << c if bar
       if c == "\n"
+        bar.process(line) if bar
         if pid
           Log.logn "STDOUT [#{pid}]: ", level
         else
           Log.logn "STDOUT: ", level
         end
+        line = "" if bar
       end
     end 
-    io.join
+    begin
+      io.join
+      bar.remove if bar
+    rescue
+      bar.remove(true) if bar
+      raise $!
+    end
 
     nil
   end

data/lib/rbbt/util/config.rb

@@ -90,6 +90,8 @@ module Rbbt::Config
     options = tokens.pop if Hash === tokens.last
     default = options.nil? ? nil : options[:default]
 
+    tokens = ["key:" + key] if tokens.empty?
+
     tokens = tokens.flatten
     file, _sep, line = caller.reject{|l| 
       l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
@@ -106,7 +108,6 @@ module Rbbt::Config
 
     entries = CACHE[key.to_s]
     priorities = {}
-    tokens = tokens + ["key:" << key.to_s]
     tokens.each do |token|
       token_prio = match entries, token.to_s
       token_prio.each do |prio, values|
@@ -145,7 +146,6 @@ module Rbbt::Config
       Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
     else
       key, value, *tokens = config.split(/\s/)
-      tokens = ['key:' << key << '::0'] if tokens.empty?
       tokens = tokens.collect do |tok|
         tok, _sep, prio = tok.partition("::")
         prio = "0" if prio.nil? or prio.empty?

data/lib/rbbt/util/log/progress.rb

@@ -18,11 +18,11 @@ module Log
       attr_accessor :default_file
     end
 
-    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes
+    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback
 
     def initialize(max = nil, options = {})
       options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
-      depth, num_reports, desc, io, severity, file, bytes, frequency = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency
+      depth, num_reports, desc, io, severity, file, bytes, frequency, process, callback = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency, :process, :callback
 
       @max = max
       @ticks = 0
@@ -34,6 +34,8 @@ module Log
       @desc = desc.nil? ? "" : desc.gsub(/\n/,' ')
       @file = file
       @bytes = bytes
+      @process = process
+      @callback = callback
     end
 
     def percent
@@ -80,5 +82,18 @@ module Log
       step = pos - (@ticks || 0)
       tick(step)
     end
+
+    def process(elem)
+      case res = @process.call(elem)
+      when FalseClass
+        nil
+      when TrueClass
+        tick
+      when Integer
+        pos(res)
+      when Float
+        pos(res * max)
+      end
+    end
   end
 end

data/lib/rbbt/util/log/progress/report.rb

@@ -124,6 +124,29 @@ module Log
       str
     end
 
+    def load(info)
+      info.each do |key, value| 
+        case key.to_sym
+        when :start 
+          @start = value
+        when :last_time 
+          @last_time = value
+        when :last_count 
+          @last_count = value
+        when :last_percent 
+          @last_percent = value
+        when :desc 
+          @desc = value
+        when :ticks 
+          @ticks = value
+        when :max 
+          @max = value
+        when :mean 
+          @mean = value
+        end
+      end
+    end
+
     def save
       info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
       info.delete_if{|k,v| v.nil?}
@@ -154,7 +177,7 @@ module Log
       bars << self unless BARS.include? self
 
       print(io, Log.up_lines(bars.length) << Log.color(:magenta, "···Progress\n") << Log.down_lines(bars.length+1)) if Log::ProgressBar.offset == 0
-      print(io, Log.up_lines(@depth) << report_msg << Log.down_lines(@depth)) 
+      print(io, Log.up_lines(@depth) << report_msg << "\n" << Log.down_lines(@depth - 1)) 
       @last_time = Time.now
       @last_count = ticks
       @last_percent = percent if max and max > 0
@@ -175,7 +198,10 @@ module Log
       done_msg << " - " << thr_msg 
       done_msg << Log.color(:magenta, " · " << desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
-      Open.rm file if file and Open.exists? file
+
+      Open.rm file if file and Open.exists?(file)
+
+      @callback.call self if @callback
     end
 
     def error(io = STDERR)
@@ -192,7 +218,14 @@ module Log
       done_msg << " - " << thr_msg
       done_msg << Log.color(:magenta, " · " << desc)      
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
-      Open.rm file if file and Open.exists? file
+
+      Open.rm file if file and Open.exists?(file)
+
+      begin
+        @callback.call self
+      rescue
+        Log.debug "Callback failed for filed progress bar: #{$!.message}"
+      end if @callback
     end
   end
 end

data/lib/rbbt/util/misc/omics.rb

@@ -312,11 +312,23 @@ module Misc
     end
   end
 
+  def self.sort_genomic_locations_by_contig(stream, contigs, sep = ":")
+    ext_stream = TSV.traverse stream, :type => :array, :into => :stream do |line|
+      chr = line.partition(sep).first
+      num = contigs.index chr
+      num.to_s + sep + line
+    end
+
+    TSV.traverse sort_stream(ext_stream, '#', "-k1,1n -k3,3n -t#{sep}"), :type => :array, :into => :stream do |line|
+      line.partition(sep).last
+    end
+  end
+  
   def self.sort_genomic_locations_strict(stream, sep = ":")
     sort_stream(stream, '#', "-k1,1V -k2,2n -t#{sep}")
   end
   
-  def self.sort_genomic_locations(stream)
+  def self.sort_genomic_locations(stream, sep = ":")
     sort_stream(stream, '#', "-k1,1 -k2,2n -t#{sep}")
   end
 
@@ -386,6 +398,7 @@ module Misc
       cmp
     end
   end
+
   def self.intersect_streams_cmp_chr(chr1, chr2)
     chr1 <=> chr2
   end
@@ -492,4 +505,50 @@ module Misc
       end
     end
   end
+
+  def self.genomic_mutations_to_BED(mutations, chr_prefix = false, sort_order = :normal)
+    io = if Array === sort_order
+
+           case chr_prefix.to_s.downcase
+           when "remove"
+             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
+           when "true", "add"
+             sort_order = sort_order.collect{|chr| chr.sub('chr', '') }  if sort_order.first.include?('chr') 
+           end
+
+           sort_genomic_locations_by_contig(mutations, sort_order)
+
+         else
+
+           case sort_order.to_s
+           when 'strict'
+             sort_genomic_locations_strict(mutations)
+           else
+             sort_genomic_locations(mutations)
+           end
+
+         end
+
+    TSV.traverse io, :type => :array, :into => :stream do |mutation|
+      chr, pos, mut, *rest = mutation.split(":")
+      size = case mut
+             when nil
+               1
+             when /^\+(.*)/
+               1 + $1.length
+             when /^\-(.*)/
+               $1.length
+             else
+               mut.length
+             end
+
+      case chr_prefix.to_s.downcase
+      when "true", "add"
+        chr = "chr" + chr if ! chr.include?('chr')
+      when "remove"
+        chr = chr.sub("chr", '') if chr.include?('chr')
+      end
+      [chr, pos.to_i - 1, pos.to_i - 1 + size, mutation] * "\t"
+    end
+  end
 end

data/lib/rbbt/workflow.rb

@@ -379,7 +379,7 @@ module Workflow
     has_overriden_inputs = false
 
     inputs.each do |k,v|
-      has_overriden_inputs = true if String === k and k.include? "#"
+      #has_overriden_inputs = true if String === k and k.include? "#"
       next unless (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s))
       default = all_defaults[k]
       next if default == v 

data/lib/rbbt/workflow/accessor.rb

@@ -250,6 +250,8 @@ module Workflow
   end
 
   def assign_dep_inputs(_inputs, options, all_d, task_info)
+    IndiferentHash.setup(_inputs)
+
     options.each{|i,v|
       next if i == :compute or i == "compute"
       case v
@@ -259,13 +261,16 @@ module Workflow
         rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
 
         if rec_dependency.nil?
-          if _inputs.include? v
-            _inputs[i] = _inputs.delete(v)
+          if _inputs.include?(v)
+            #_inputs[i] = _inputs.delete(v)
+            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
           else
             _inputs[i] = v unless _inputs.include? i
           end
         else
           input_options = task_info[:input_options][i] || {}
+
+          #ToDo why was this always true?
           if input_options[:stream] or true
             #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
             _inputs[i] = rec_dependency

data/lib/rbbt/workflow/definition.rb

@@ -79,9 +79,10 @@ module Workflow
       dep = dependencies.last.join
       raise dep.get_exception if dep.error?
       set_info :result_type, dep.info[:result_type]
-      forget = config :forget_dep_tasks, :forget_dep_tasks, :default => FORGET_DEP_TASKS
+      forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
-        remove = config :remove_dep_tasks, :remove_dep_tasks, :default => REMOVE_DEP_TASKS
+        remove = config :remove_dep_tasks, "remove_dep_tasks", "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
+
         self.archive_deps
         self.copy_files_dir
         self.dependencies = self.dependencies - [dep]
@@ -92,7 +93,10 @@ module Workflow
         when 'true'
           dep.clean
         when 'recursive'
-          dep.recursive_clean
+          dep.rec_dependencies.each do |d|
+            d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
+          end
+          dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
         end
       else
         if Open.exists?(dep.files_dir)

data/lib/rbbt/workflow/step.rb

@@ -114,9 +114,13 @@ class Step
 
   def copy_files_dir
     if File.symlink?(self.files_dir)
-      realpath = Open.realpath(self.files_dir)
-      Open.rm self.files_dir
-      Open.cp realpath, self.files_dir
+      begin
+        realpath = Open.realpath(self.files_dir)
+        Open.rm self.files_dir
+        Open.cp realpath, self.files_dir
+      rescue
+        Log.warn "Copy files_dir for #{self.workflow_short_path}: " + $!.message
+      end
     end
   end
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -168,6 +168,10 @@ class Step
     @task_name ||= task.name
   end
 
+  def task_signature
+    [workflow.to_s, task_name] * "#"
+  end
+
   # {{{ INFO
 
   def info_file

data/lib/rbbt/workflow/step/run.rb

@@ -623,6 +623,15 @@ class Step
         Log.warn "Exception removing result of aborted job: #{$!.message}"
       end
     end
+
+    if Open.exists?(tmp_path) && status != :done
+      Log.warn "Aborted job had finished. Removing tmp result -- #{ tmp_path }"
+      begin
+        Open.rm tmp_path
+      rescue Exception
+        Log.warn "Exception removing tmp result of aborted job: #{$!.message}"
+      end
+    end
   end
 
   def _abort

data/lib/rbbt/workflow/util/archive.rb

@@ -51,7 +51,7 @@ class Step
     end
   end
 
-  def self.job_files_for_archive(files, recursive = false)
+  def self.job_files_for_archive(files, recursive = false, skip_overriden = false)
     job_files = Set.new
 
     jobs = files.collect do |file|  
@@ -65,6 +65,8 @@ class Step
 
     jobs.each do |step|
       next unless File.exists?(step.path)
+      next if skip_overriden && step.overriden
+
       job_files << step.path
       job_files << step.info_file if File.exists?(step.info_file)
       job_files << Step.md5_file(step.path) if File.exists?(Step.md5_file step.path)
@@ -258,9 +260,9 @@ puts resource[path].find(search_path)
     end
   end
 
-  def self.purge(path, recursive = false)
+  def self.purge(path, recursive = false, skip_overriden = true)
     path = [path] if String === path
-    job_files = job_files_for_archive path, recursive
+    job_files = job_files_for_archive path, recursive, skip_overriden
 
     job_files.each do |file|
       begin

data/lib/rbbt/workflow/util/provenance.rb

@@ -1,23 +1,28 @@
 class Step
+
+  def self.status_color(status)
+    case status.to_sym
+    when :error, :aborted, :missing, :dead, :unsync
+      :red
+    when :streaming, :started
+      :cyan
+    when :done, :noinfo
+      :green
+    when :dependencies, :waiting, :setup
+      :yellow
+    when :notfound, :cleaned
+      :blue
+    else
+      if status.to_s.index ">"
+        :cyan
+      else
+        :cyan
+      end
+    end
+  end
+
   def self.prov_status_msg(status)
-    color = case status.to_sym
-            when :error, :aborted, :missing, :dead, :unsync
-              :red
-            when :streaming, :started
-              :cyan
-            when :done, :noinfo
-              :green
-            when :dependencies, :waiting, :setup
-              :yellow
-            when :notfound, :cleaned
-              :blue
-            else
-              if status.to_s.index ">"
-                :cyan
-              else
-                :cyan
-              end
-            end
+    color = status_color(status)
     Log.color(color, status.to_s)
   end
 
@@ -25,7 +30,7 @@ class Step
     parts = path.sub(/\{.*/,'').split "/"
 
     parts.pop
-    
+
     task = Log.color(:yellow, parts.pop)
     workflow = Log.color(:magenta, parts.pop)
     #if status.to_s == 'noinfo' && parts.last != 'jobs'
@@ -89,7 +94,7 @@ class Step
         str << prov_report(dep, offset + 1, task, seen, expand_repeats)
       else
         if expand_repeats
-          str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
         else
           info = dep.info  || {}
           status = info[:status] || :missing
@@ -98,7 +103,7 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(status == :notfound ? :blue : :green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
         end
       end
     end if step.dependencies

data/share/Rlib/util.R

@@ -484,7 +484,7 @@ rbbt.model.inpute <- function(data, formula, ...){
     data
 }
 
-rbbt.tsv.melt <- function(tsv, variable = NULL, value = NULL, key.field = NULL){
+rbbt.tsv.melt <- function(tsv, variable = NULL, value = NULL, key.field = NULL, ...){
     rbbt.require('reshape2')
     if (is.null(key.field)){ key.field = attributes(data)$key.field;}
     if (is.null(key.field)){ key.field = "ID" }
@@ -494,7 +494,7 @@ rbbt.tsv.melt <- function(tsv, variable = NULL, value = NULL, key.field = NULL){
 
     tsv[key.field] = rownames(tsv)
 
-    m <- melt(tsv)
+    m <- melt(tsv, id.vars=c(key.field), ...)
 
     names(m) <- c(key.field, variable, value)
 
@@ -784,6 +784,12 @@ rbbt.plot.venn <- function(data, a=NULL, category=NULL, fill=NULL, ...) {
     return(out)
 }
 
+rbbt.plot.upset <- function(data, variable = NULL, ...){
+  rbbt.require('UpSetR')
+  data[data == TRUE] = 1
+  return(upset(data, ...))
+}
+
 rbbt.plot.pca <- function(data, center = TRUE, scale. = TRUE, ...) {
   rbbt.require('vqv/ggbiplot')
   data <- rbbt.impute(data)

data/share/rbbt_commands/slurm/list

@@ -20,6 +20,7 @@ $ rbbt mnl [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-p--progress Report progress of job and the dependencies
 -SBP--sbatch_parameters show sbatch parameters
 -PERF--procpath_performance show Procpath performance summary
 -sacct--sacct_peformance show sacct performance summary
@@ -34,8 +35,8 @@ if options[:help]
  exit 0
 end
 
-Log.severity = 4
-done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
+#Log.severity = 4
+done, error, running, queued, aborted, jobid, search, tail, progress = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail, :progress
 
 workdir = File.expand_path('~/rbbt-slurm')
 Path.setup(workdir)
@@ -228,13 +229,30 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  if tail && File.exists?(File.join(dir, 'std.err'))
    if exit_status && exit_status != 0
      puts Log.color(:magenta, "First error or exception found: ")
-     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} | grep -v '^\\S\\[3.m' |head -n #{tail.to_i}", :no_fail => true).read
    elsif exit_status
      puts Log.color(:magenta, "Completed jobs: ")
-     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | grep -v '^\\S\\[3.m' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^\\S*\\[3.m'  | tail -n #{tail.to_i} ").read
+   end
+ end
+
+ if options[:progress] 
+   step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+   if step_line
+     require 'rbbt/workflow'
+     step_path = step_line.split(": ").last.strip
+     step = Step.new step_path
+     step.load_dependencies_from_info
+     (step.rec_dependencies + [step]).reverse.each do |j|
+       next if j.done? || ! j.running?
+       next unless j.file(:progress).exists?
+       bar = Log::ProgressBar.new 
+       bar.load(j.file(:progress).yaml)
+       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+     end
    end
  end
 

data/share/rbbt_commands/workflow/server

@@ -16,6 +16,8 @@ $ rbbt workflow server [options] <Workflow>
 -B--Bind* Bind IP
 -p--port* TCP port
 -s--server* Server type:  thin, webrick, unicorn, etc
+-w--workers* Number of workers for cluster mode (puma)
+-so--server_options* Additional options for server (e.g. option1=value1;option2=value2)
 -f--finder Start server with finder functionality
 -RS--Rserve_session* Rserve session to use, otherwise start new one
 -wd--workdir* Change the working directory of the workflow
@@ -32,7 +34,6 @@ $ rbbt workflow server [options] <Workflow>
 --export_synchronous* Export workflow tasks as synchronous
 --export_exec* Export workflow tasks as exec
 --export_stream* Export workflow tasks as stream
---options* Additional options for server (e.g. option1=value1;option2=value2)
 EOF
 
 if options[:help]
@@ -108,20 +109,24 @@ TmpFile.with_file do |app_dir|
     config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
 
 
-    if options[:options]
-      options[:options].split(";").each do |pair|
+    if options[:server_options]
+      options[:server_options].split(";").each do |pair|
         name, _sep, value = pair.partition("=")
         name = name[1..-1].to_sym if name[0] == ':'
+        value = value.to_i if value =~ /^\d+$/
+        value = true if value == "true"
+        value = false if value == "false"
         options[name] = value
       end
+      options.delete :server_options
     end
 
     case server
     when 'unicorn'
       `env RBBT_LOG=#{Log.severity.to_s} unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:Port] || "2887"}`
-    when 'puma_alt'
+    when 'puma_production'
       #`puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload`
-      `env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 20 -t 10:160 --preload`
+      CMD.cmd_log("env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 20 -t 10:160 --preload")
     else
       options[:config] = config_ru_file
 

data/test/rbbt/util/test_config.rb

@@ -5,8 +5,9 @@ class TestConfig < Test::Unit::TestCase
   def setup
     Rbbt::Config.set({:cpus => 30}, :test_config, :test)
     Rbbt::Config.set(:cpus, 5, "slow::2", :test)
-    Rbbt::Config.set({:token => "token"}, "token", "key:token")
+    Rbbt::Config.set({:token => "token"}, "token")
     Rbbt::Config.set(:notoken, "no_token")
+    Rbbt::Config.set({:emptytoken => "empty"})
   end
 
   def test_simple
@@ -19,8 +20,12 @@ class TestConfig < Test::Unit::TestCase
 
   def test_simple_no_token
     assert_equal "token", Rbbt::Config.get("token", "token")
-    assert_equal "token", Rbbt::Config.get("token")
-    assert_equal "no_token", Rbbt::Config.get("notoken")
+    assert_equal "no_token", Rbbt::Config.get("notoken", "key:notoken")
+    assert_equal 'token', Rbbt::Config.get("token", "key:token")
+    assert_equal 'token', Rbbt::Config.get("token")
+    assert_equal nil, Rbbt::Config.get("token", "someotherthing")
+    assert_equal "default_token", Rbbt::Config.get("token", 'unknown', :default => 'default_token')
+    assert_equal 'empty', Rbbt::Config.get("emptytoken", 'key:emptytoken')
   end
 
   def test_prio
@@ -54,6 +59,11 @@ class TestConfig < Test::Unit::TestCase
     assert_equal "V1", Rbbt::Config.get('key', 'token1', 'token2')
   end
 
+  def test_default
+    Rbbt::Config.add_entry 'key', 'V1', 'token1'
+    assert_equal "V3", Rbbt::Config.get('key', 'token2', :default => 'V3')
+  end
+
 
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.30.6
+  version: 5.30.11
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-09 00:00:00.000000000 Z
+date: 2021-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake