rbbt-util 5.30.10 → 5.30.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48bec783cd5a61ffb37cb7cd80379ed9c90bbcfd2cecdb756cff0f2efd6240ed
-  data.tar.gz: 9da9e93fc5c89be3ca8f57f87ca2808f298e6a300cd13dfff4dd602a2bf1b4b0
+  metadata.gz: '009c7b1e63d04078314d77556ca0a68c9a29d875a5cd69ff57335650bd20823b'
+  data.tar.gz: 9a6fb144f833fff84a234b226a076e24355ab3733500b09c1a96bb1d80d004f8
 SHA512:
-  metadata.gz: 23976e1fcc0a216a822fa19ec9f4510d87f5d19d334245f6cea87746d3b7b85cf3c41110ce2d36f3a793f6e5af1bd08ad2299ad0acb08f11efe711f3417da590
-  data.tar.gz: 2fe9f8dbea015d105076926f0407c1029a400f6eafce7f17edca075e4975ef5a4bd455f416cad4e2d4deadb2d7486988c00500687fc41352020ba71fa07a0f0c
+  metadata.gz: 58bd3482675c9e55b71fe1b6fa79dec4f23ac40d321067c19412c762d8de30983918ab9866869cb2a93c590e1f2a5f7c4228c4ba54d1c4e247a627d5948707db
+  data.tar.gz: 0f7f3c371fd6dcacf0050fcaf5e8ea3da81e6665b7640bbc596a718a386a7fd563e89bd35189a8ccb3c45e0badcf52bc37aa585b18753a31ae5178c51c0dcd3c

data/lib/rbbt/hpc/orchestrate.rb

@@ -67,7 +67,7 @@ module HPC
     def self.piggyback(job, job_rules, job_deps)
       return false unless job_rules["skip"]
       final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
-      final_deps = final_deps.reject{|dep| dep.done? && dep.status == :noinfo }
+      final_deps = final_deps.reject{|dep| dep.done? }
       return final_deps.first if final_deps.length == 1
       return false
     end

data/lib/rbbt/hpc/slurm.rb

@@ -21,6 +21,8 @@ module HPC
       exclusive        = options.delete :exclusive
       highmem          = options.delete :highmem
 
+      slurm_step_path  = options.delete :slurm_step_path
+
       manifest         = options.delete :manifest
 
       queue            = options.delete(:queue) || Rbbt::Config.get('queue', :slurm_queue, :slurm, :SLURM, :default => 'bsc_ls')
@@ -264,6 +266,10 @@ EOF
 #MANIFEST: #{manifest * ", "}
       EOF
 
+      header +=<<-EOF if slurm_step_path
+#STEP_PATH: #{slurm_step_path}
+      EOF
+
       header +=<<-EOF
 #CMD: #{rbbt_cmd}
       EOF
@@ -536,7 +542,9 @@ EOF
       dependencies     = options.delete :slurm_dependencies
       procpath         = options.delete :SLURM_procpath
 
-      options[:jobname] = job.clean_name
+      options[:jobname]   = job.clean_name
+      options[:slurm_step_path] = job.path
+
       log_level         = options.delete :log
       log_level       ||= Log.severity
 

data/lib/rbbt/util/cmd.rb

@@ -101,6 +101,7 @@ module CMD
     no_fail    = options.delete(:nofail) if no_fail.nil?
     no_wait    = options.delete(:no_wait)
     xvfb       = options.delete(:xvfb)
+    bar        = options.delete(:progress_bar)
 
     dont_close_in  = options.delete(:dont_close_in)
 
@@ -183,6 +184,7 @@ module CMD
 
       err_thread = Thread.new do
         while line = serr.gets
+          bar.process(line) if bar
           sout.log = line
           Log.log "STDERR [#{pid}]: " +  line, stderr 
         end if Integer === stderr and log
@@ -220,6 +222,8 @@ module CMD
   def self.cmd_pid(*args)
     all_args = *args
 
+    bar = all_args.last[:progress_bar] if Hash === all_args.last
+
     all_args << {} unless Hash === all_args.last
 
     level = all_args.last[:log] || 0
@@ -233,17 +237,27 @@ module CMD
     io = cmd(*all_args)
     pid = io.pids.first
 
+    line = "" if bar
     while c = io.getc
       STDERR << c if Log.severity <= level
+      line << c if bar
       if c == "\n"
+        bar.process(line) if bar
         if pid
           Log.logn "STDOUT [#{pid}]: ", level
         else
           Log.logn "STDOUT: ", level
         end
+        line = "" if bar
       end
     end 
-    io.join
+    begin
+      io.join
+      bar.remove if bar
+    rescue
+      bar.remove(true) if bar
+      raise $!
+    end
 
     nil
   end

data/lib/rbbt/util/log/progress.rb

@@ -18,11 +18,11 @@ module Log
       attr_accessor :default_file
     end
 
-    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes
+    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback
 
     def initialize(max = nil, options = {})
       options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
-      depth, num_reports, desc, io, severity, file, bytes, frequency = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency
+      depth, num_reports, desc, io, severity, file, bytes, frequency, process, callback = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency, :process, :callback
 
       @max = max
       @ticks = 0
@@ -34,6 +34,8 @@ module Log
       @desc = desc.nil? ? "" : desc.gsub(/\n/,' ')
       @file = file
       @bytes = bytes
+      @process = process
+      @callback = callback
     end
 
     def percent
@@ -80,5 +82,18 @@ module Log
       step = pos - (@ticks || 0)
       tick(step)
     end
+
+    def process(elem)
+      case res = @process.call(elem)
+      when FalseClass
+        nil
+      when TrueClass
+        tick
+      when Integer
+        pos(res)
+      when Float
+        pos(res * max)
+      end
+    end
   end
 end

data/lib/rbbt/util/log/progress/report.rb

@@ -124,6 +124,29 @@ module Log
       str
     end
 
+    def load(info)
+      info.each do |key, value| 
+        case key.to_sym
+        when :start 
+          @start = value
+        when :last_time 
+          @last_time = value
+        when :last_count 
+          @last_count = value
+        when :last_percent 
+          @last_percent = value
+        when :desc 
+          @desc = value
+        when :ticks 
+          @ticks = value
+        when :max 
+          @max = value
+        when :mean 
+          @mean = value
+        end
+      end
+    end
+
     def save
       info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
       info.delete_if{|k,v| v.nil?}
@@ -154,7 +177,7 @@ module Log
       bars << self unless BARS.include? self
 
       print(io, Log.up_lines(bars.length) << Log.color(:magenta, "···Progress\n") << Log.down_lines(bars.length+1)) if Log::ProgressBar.offset == 0
-      print(io, Log.up_lines(@depth) << report_msg << Log.down_lines(@depth)) 
+      print(io, Log.up_lines(@depth) << report_msg << "\n" << Log.down_lines(@depth - 1)) 
       @last_time = Time.now
       @last_count = ticks
       @last_percent = percent if max and max > 0
@@ -175,7 +198,10 @@ module Log
       done_msg << " - " << thr_msg 
       done_msg << Log.color(:magenta, " · " << desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
-      Open.rm file if file and Open.exists? file
+
+      Open.rm file if file and Open.exists?(file)
+
+      @callback.call self if @callback
     end
 
     def error(io = STDERR)
@@ -192,7 +218,14 @@ module Log
       done_msg << " - " << thr_msg
       done_msg << Log.color(:magenta, " · " << desc)      
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
-      Open.rm file if file and Open.exists? file
+
+      Open.rm file if file and Open.exists?(file)
+
+      begin
+        @callback.call self
+      rescue
+        Log.debug "Callback failed for filed progress bar: #{$!.message}"
+      end if @callback
     end
   end
 end

data/lib/rbbt/util/misc/omics.rb

@@ -312,11 +312,23 @@ module Misc
     end
   end
 
+  def self.sort_genomic_locations_by_contig(stream, contigs, sep = ":")
+    ext_stream = TSV.traverse stream, :type => :array, :into => :stream do |line|
+      chr = line.partition(sep).first
+      num = contigs.index chr
+      num.to_s + sep + line
+    end
+
+    TSV.traverse sort_stream(ext_stream, '#', "-k1,1n -k3,3n -t#{sep}"), :type => :array, :into => :stream do |line|
+      line.partition(sep).last
+    end
+  end
+  
   def self.sort_genomic_locations_strict(stream, sep = ":")
     sort_stream(stream, '#', "-k1,1V -k2,2n -t#{sep}")
   end
   
-  def self.sort_genomic_locations(stream)
+  def self.sort_genomic_locations(stream, sep = ":")
     sort_stream(stream, '#', "-k1,1 -k2,2n -t#{sep}")
   end
 
@@ -386,6 +398,7 @@ module Misc
       cmp
     end
   end
+
   def self.intersect_streams_cmp_chr(chr1, chr2)
     chr1 <=> chr2
   end
@@ -492,4 +505,50 @@ module Misc
       end
     end
   end
+
+  def self.genomic_mutations_to_BED(mutations, chr_prefix = false, sort_order = :normal)
+    io = if Array === sort_order
+
+           case chr_prefix.to_s.downcase
+           when "remove"
+             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
+           when "true", "add"
+             sort_order = sort_order.collect{|chr| chr.sub('chr', '') }  if sort_order.first.include?('chr') 
+           end
+
+           sort_genomic_locations_by_contig(mutations, sort_order)
+
+         else
+
+           case sort_order.to_s
+           when 'strict'
+             sort_genomic_locations_strict(mutations)
+           else
+             sort_genomic_locations(mutations)
+           end
+
+         end
+
+    TSV.traverse io, :type => :array, :into => :stream do |mutation|
+      chr, pos, mut, *rest = mutation.split(":")
+      size = case mut
+             when nil
+               1
+             when /^\+(.*)/
+               1 + $1.length
+             when /^\-(.*)/
+               $1.length
+             else
+               mut.length
+             end
+
+      case chr_prefix.to_s.downcase
+      when "true", "add"
+        chr = "chr" + chr if ! chr.include?('chr')
+      when "remove"
+        chr = chr.sub("chr", '') if chr.include?('chr')
+      end
+      [chr, pos.to_i - 1, pos.to_i - 1 + size, mutation] * "\t"
+    end
+  end
 end

data/lib/rbbt/workflow/accessor.rb

@@ -250,6 +250,8 @@ module Workflow
   end
 
   def assign_dep_inputs(_inputs, options, all_d, task_info)
+    IndiferentHash.setup(_inputs)
+
     options.each{|i,v|
       next if i == :compute or i == "compute"
       case v
@@ -259,13 +261,16 @@ module Workflow
         rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
 
         if rec_dependency.nil?
-          if _inputs.include? v
-            _inputs[i] = _inputs.delete(v)
+          if _inputs.include?(v)
+            #_inputs[i] = _inputs.delete(v)
+            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
           else
             _inputs[i] = v unless _inputs.include? i
           end
         else
           input_options = task_info[:input_options][i] || {}
+
+          #ToDo why was this always true?
           if input_options[:stream] or true
             #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
             _inputs[i] = rec_dependency

data/lib/rbbt/workflow/definition.rb

@@ -79,9 +79,9 @@ module Workflow
       dep = dependencies.last.join
       raise dep.get_exception if dep.error?
       set_info :result_type, dep.info[:result_type]
-      forget = config :forget_dep_tasks, "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
+      forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
-        remove = config :remove_dep_tasks, "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
+        remove = config :remove_dep_tasks, "remove_dep_tasks", "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
 
         self.archive_deps
         self.copy_files_dir

data/lib/rbbt/workflow/step/run.rb

@@ -623,6 +623,15 @@ class Step
         Log.warn "Exception removing result of aborted job: #{$!.message}"
       end
     end
+
+    if Open.exists?(tmp_path) && status != :done
+      Log.warn "Aborted job had finished. Removing tmp result -- #{ tmp_path }"
+      begin
+        Open.rm tmp_path
+      rescue Exception
+        Log.warn "Exception removing tmp result of aborted job: #{$!.message}"
+      end
+    end
   end
 
   def _abort

data/lib/rbbt/workflow/util/provenance.rb

@@ -1,23 +1,28 @@
 class Step
+
+  def self.status_color(status)
+    case status.to_sym
+    when :error, :aborted, :missing, :dead, :unsync
+      :red
+    when :streaming, :started
+      :cyan
+    when :done, :noinfo
+      :green
+    when :dependencies, :waiting, :setup
+      :yellow
+    when :notfound, :cleaned
+      :blue
+    else
+      if status.to_s.index ">"
+        :cyan
+      else
+        :cyan
+      end
+    end
+  end
+
   def self.prov_status_msg(status)
-    color = case status.to_sym
-            when :error, :aborted, :missing, :dead, :unsync
-              :red
-            when :streaming, :started
-              :cyan
-            when :done, :noinfo
-              :green
-            when :dependencies, :waiting, :setup
-              :yellow
-            when :notfound, :cleaned
-              :blue
-            else
-              if status.to_s.index ">"
-                :cyan
-              else
-                :cyan
-              end
-            end
+    color = status_color(status)
     Log.color(color, status.to_s)
   end
 
@@ -25,7 +30,7 @@ class Step
     parts = path.sub(/\{.*/,'').split "/"
 
     parts.pop
-    
+
     task = Log.color(:yellow, parts.pop)
     workflow = Log.color(:magenta, parts.pop)
     #if status.to_s == 'noinfo' && parts.last != 'jobs'
@@ -89,7 +94,7 @@ class Step
         str << prov_report(dep, offset + 1, task, seen, expand_repeats)
       else
         if expand_repeats
-          str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
         else
           info = dep.info  || {}
           status = info[:status] || :missing
@@ -98,7 +103,7 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(status == :notfound ? :blue : :green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
         end
       end
     end if step.dependencies

data/share/rbbt_commands/slurm/list

@@ -20,6 +20,7 @@ $ rbbt mnl [options]
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
+-p--progress Report progress of job and the dependencies
 -SBP--sbatch_parameters show sbatch parameters
 -PERF--procpath_performance show Procpath performance summary
 -sacct--sacct_peformance show sacct performance summary
@@ -34,8 +35,8 @@ if options[:help]
  exit 0
 end
 
-Log.severity = 4
-done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
+#Log.severity = 4
+done, error, running, queued, aborted, jobid, search, tail, progress = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail, :progress
 
 workdir = File.expand_path('~/rbbt-slurm')
 Path.setup(workdir)
@@ -228,13 +229,30 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  if tail && File.exists?(File.join(dir, 'std.err'))
    if exit_status && exit_status != 0
      puts Log.color(:magenta, "First error or exception found: ")
-     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} | grep -v '^\\S\\[3.m' |head -n #{tail.to_i}", :no_fail => true).read
    elsif exit_status
      puts Log.color(:magenta, "Completed jobs: ")
-     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | grep -v '^\\S\\[3.m' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^\\S*\\[3.m'  | tail -n #{tail.to_i} ").read
+   end
+ end
+
+ if options[:progress] 
+   step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+   if step_line
+     require 'rbbt/workflow'
+     step_path = step_line.split(": ").last.strip
+     step = Step.new step_path
+     step.load_dependencies_from_info
+     (step.rec_dependencies + [step]).reverse.each do |j|
+       next if j.done? || ! j.running?
+       next unless j.file(:progress).exists?
+       bar = Log::ProgressBar.new 
+       bar.load(j.file(:progress).yaml)
+       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+     end
    end
  end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.30.10
+  version: 5.30.11
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-16 00:00:00.000000000 Z
+date: 2021-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake