rbbt-util 5.30.9 → 5.31.0

data/lib/rbbt/resource/path.rb

@@ -3,7 +3,7 @@ require 'rbbt/util/misc/indiferent_hash'
 require 'yaml'
 
 module Path
-  attr_accessor :resource, :pkgdir, :original, :search_paths, :search_order, :libdir
+  attr_accessor :resource, :pkgdir, :original, :search_paths, :search_order, :libdir, :where
 
   def self.setup(string, pkgdir = nil, resource = nil, search_paths = nil, search_order = nil, libdir = nil)
     return string if string.nil?
@@ -99,6 +99,7 @@ module Path
 
       paths = paths.each do |p|
         p.original = File.join(found.original, p.sub(/^#{found}/, ''))
+        p.where = where
       end if found.original and pattern
 
       location_paths[where] = paths
@@ -256,6 +257,7 @@ module Path
                      end
 
                      res.original = self.original || self
+                     res.where = where
 
                      res
                    end

data/lib/rbbt/tsv/accessor.rb

@@ -713,8 +713,11 @@ module TSV
       break
     end
 
-    filename = Path === filename ? filename.find : (filename || "No filename")
-    filename + " [" + persistence_path + "]" if respond_to?(:persistence_path) and persistence_path
+    filename = @filename
+    filename = "No filename" if filename.nil? || filename.empty?
+    filename.find if Path === filename 
+    filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
+
     with_unnamed do
       <<-EOF
 Filename = #{filename}

data/lib/rbbt/tsv/dumper.rb

@@ -78,6 +78,7 @@ module TSV
     end
 
     def close_in
+      @in_stream.join if @in_stream.respond_to?(:join) && ! @in_stream.joined?
       @in_stream.close unless @in_stream.closed?
     end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -610,7 +610,7 @@ module TSV
     thread = Thread.new do 
       begin
         traverse_run(obj, threads, cpus, options, &block)
-        into.close if into.respond_to?(:close) and not (into.respond_to? :closed? and into.closed?) 
+        into.close if into.respond_to?(:close) and not (into.respond_to?(:closed?) and into.closed?) 
       rescue Exception
         abort_stream obj
         abort_stream into

data/lib/rbbt/tsv/stream.rb

@@ -294,20 +294,19 @@ module TSV
   end
 
 
-  def self.reorder_stream_tsv(stream, key_field, fields=nil, zipped = true)
+  def self.reorder_stream_tsv(stream, key_field, fields=nil, zipped = true, bar = nil)
     parser = TSV::Parser.new TSV.get_stream(stream), :key_field => key_field, :fields => fields
     dumper_options = parser.options
     dumper = TSV::Dumper.new dumper_options
     dumper.init 
     case parser.type
     when :single
-      TSV.traverse parser, :into => dumper do |keys,values|
+      TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
         key = keys.first
         [key, [values]]
       end
     when :double
-      TSV.traverse parser, :into => dumper do |keys,values|
-        raise [keys, values].inspect if keys.include? 'gain'
+      TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
         res = []
         keys.each_with_index do |key,i|
           vs = zipped ?  values.collect{|l| l.length == 1 ? l : [l[i]] } : values
@@ -317,12 +316,12 @@ module TSV
         res
       end
     when :list
-      TSV.traverse parser, :into => dumper do |keys,values|
+      TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
         key = keys === Array ? keys.first : keys
         [key, values]
       end
     when :flat
-      TSV.traverse parser, :into => dumper do |keys,values|
+      TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
         key = keys === Array ? keys.first : keys
         [key, values]
       end

data/lib/rbbt/util/cmd.rb

@@ -101,6 +101,7 @@ module CMD
     no_fail    = options.delete(:nofail) if no_fail.nil?
     no_wait    = options.delete(:no_wait)
     xvfb       = options.delete(:xvfb)
+    bar        = options.delete(:progress_bar)
 
     dont_close_in  = options.delete(:dont_close_in)
 
@@ -183,6 +184,7 @@ module CMD
 
       err_thread = Thread.new do
         while line = serr.gets
+          bar.process(line) if bar
           sout.log = line
           Log.log "STDERR [#{pid}]: " +  line, stderr 
         end if Integer === stderr and log
@@ -220,6 +222,8 @@ module CMD
   def self.cmd_pid(*args)
     all_args = *args
 
+    bar = all_args.last[:progress_bar] if Hash === all_args.last
+
     all_args << {} unless Hash === all_args.last
 
     level = all_args.last[:log] || 0
@@ -233,17 +237,27 @@ module CMD
     io = cmd(*all_args)
     pid = io.pids.first
 
+    line = "" if bar
     while c = io.getc
       STDERR << c if Log.severity <= level
+      line << c if bar
       if c == "\n"
+        bar.process(line) if bar
         if pid
           Log.logn "STDOUT [#{pid}]: ", level
         else
           Log.logn "STDOUT: ", level
         end
+        line = "" if bar
       end
     end 
-    io.join
+    begin
+      io.join
+      bar.remove if bar
+    rescue
+      bar.remove(true) if bar
+      raise $!
+    end
 
     nil
   end

data/lib/rbbt/util/config.rb

@@ -90,6 +90,8 @@ module Rbbt::Config
     options = tokens.pop if Hash === tokens.last
     default = options.nil? ? nil : options[:default]
 
+    tokens = ["key:" + key] if tokens.empty?
+
     tokens = tokens.flatten
     file, _sep, line = caller.reject{|l| 
       l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
@@ -106,7 +108,6 @@ module Rbbt::Config
 
     entries = CACHE[key.to_s]
     priorities = {}
-    tokens = tokens + ["key:" << key.to_s]
     tokens.each do |token|
       token_prio = match entries, token.to_s
       token_prio.each do |prio, values|
@@ -145,7 +146,6 @@ module Rbbt::Config
       Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
     else
       key, value, *tokens = config.split(/\s/)
-      tokens = ['key:' << key << '::0'] if tokens.empty?
       tokens = tokens.collect do |tok|
         tok, _sep, prio = tok.partition("::")
         prio = "0" if prio.nil? or prio.empty?

data/lib/rbbt/util/log.rb

@@ -341,10 +341,31 @@ module Log
     end unless stack.nil?
   end
 
-  def self.tsv(tsv)
+  def self.tsv(tsv, example = false)
     STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
     STDERR.puts Log.color(:blue, "=> "<< Misc.fingerprint(tsv), true) 
     STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
+    if example && ! tsv.empty?
+      key = case example
+            when TrueClass, :first, "first"
+              tsv.keys.first
+            when :random, "random"
+              tsv.keys.shuffle.first
+            else
+              example
+            end
+
+      values = tsv[key]
+      values = [values] if tsv.type == :flat || tsv.type == :single
+      if values.nil?
+        STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
+      else
+        STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
+        tsv.fields.zip(values).each do |field,value|
+          STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
+        end
+      end
+    end
   end
 
   def self.stack(stack)

data/lib/rbbt/util/log/progress.rb

@@ -18,11 +18,11 @@ module Log
       attr_accessor :default_file
     end
 
-    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes
+    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback
 
     def initialize(max = nil, options = {})
       options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
-      depth, num_reports, desc, io, severity, file, bytes, frequency = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency
+      depth, num_reports, desc, io, severity, file, bytes, frequency, process, callback = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency, :process, :callback
 
       @max = max
       @ticks = 0
@@ -34,6 +34,8 @@ module Log
       @desc = desc.nil? ? "" : desc.gsub(/\n/,' ')
       @file = file
       @bytes = bytes
+      @process = process
+      @callback = callback
     end
 
     def percent
@@ -80,5 +82,18 @@ module Log
       step = pos - (@ticks || 0)
       tick(step)
     end
+
+    def process(elem)
+      case res = @process.call(elem)
+      when FalseClass
+        nil
+      when TrueClass
+        tick
+      when Integer
+        pos(res)
+      when Float
+        pos(res * max)
+      end
+    end
   end
 end

data/lib/rbbt/util/log/progress/report.rb

@@ -124,6 +124,29 @@ module Log
       str
     end
 
+    def load(info)
+      info.each do |key, value| 
+        case key.to_sym
+        when :start 
+          @start = value
+        when :last_time 
+          @last_time = value
+        when :last_count 
+          @last_count = value
+        when :last_percent 
+          @last_percent = value
+        when :desc 
+          @desc = value
+        when :ticks 
+          @ticks = value
+        when :max 
+          @max = value
+        when :mean 
+          @mean = value
+        end
+      end
+    end
+
     def save
       info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
       info.delete_if{|k,v| v.nil?}
@@ -154,7 +177,7 @@ module Log
       bars << self unless BARS.include? self
 
       print(io, Log.up_lines(bars.length) << Log.color(:magenta, "···Progress\n") << Log.down_lines(bars.length+1)) if Log::ProgressBar.offset == 0
-      print(io, Log.up_lines(@depth) << report_msg << Log.down_lines(@depth)) 
+      print(io, Log.up_lines(@depth) << report_msg << "\n" << Log.down_lines(@depth - 1)) 
       @last_time = Time.now
       @last_count = ticks
       @last_percent = percent if max and max > 0
@@ -175,7 +198,10 @@ module Log
       done_msg << " - " << thr_msg 
       done_msg << Log.color(:magenta, " · " << desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
-      Open.rm file if file and Open.exists? file
+
+      Open.rm file if file and Open.exists?(file)
+
+      @callback.call self if @callback
     end
 
     def error(io = STDERR)
@@ -192,7 +218,14 @@ module Log
       done_msg << " - " << thr_msg
       done_msg << Log.color(:magenta, " · " << desc)      
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
-      Open.rm file if file and Open.exists? file
+
+      Open.rm file if file and Open.exists?(file)
+
+      begin
+        @callback.call self
+      rescue
+        Log.debug "Callback failed for filed progress bar: #{$!.message}"
+      end if @callback
     end
   end
 end

data/lib/rbbt/util/misc/development.rb

@@ -428,9 +428,9 @@ def self.add_libdir(dir=nil)
     end
   end
 
-
-  def self.ssh_run(server, script)
+  def self.ssh_run(server, script = nil, &block)
     Log.debug "Run ssh script in #{server}:\n#{script}"
+
     CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
   end
 

data/lib/rbbt/util/misc/inspect.rb

@@ -271,6 +271,22 @@ module Misc
   end
 
 
+  def self.step_file?(path)
+    return true if defined?(Step) && Step === path.resource
+    return false unless path.include?('.files/')
+    parts = path.split("/")
+    job = parts.select{|p| p =~ /\.files$/}.first
+    if job
+      i = parts.index job
+      begin
+        workflow, task = parts.values_at i - 2, i - 1
+        return Kernel.const_get(workflow).tasks.include? task.to_sym
+      rescue
+      end
+    end
+    false
+  end
+
   def self.obj2str(obj)
     _obj = obj
     obj = Annotated.purge(obj) if Annotated === obj
@@ -289,7 +305,7 @@ module Misc
           when (defined?(Path) and Path)
             if defined?(Step) && Open.exists?(Step.info_file(obj))
               obj2str(Workflow.load_step(obj))
-            elsif defined?(Step) && Step === obj.resource
+            elsif step_file?(obj)
               "Step file: " + obj
             else
               if obj.exists?

data/lib/rbbt/util/misc/omics.rb

@@ -312,11 +312,23 @@ module Misc
     end
   end
 
+  def self.sort_genomic_locations_by_contig(stream, contigs, sep = ":")
+    ext_stream = TSV.traverse stream, :type => :array, :into => :stream do |line|
+      chr = line.partition(sep).first
+      num = contigs.index chr
+      num.to_s + sep + line
+    end
+
+    TSV.traverse sort_stream(ext_stream, '#', "-k1,1n -k3,3n -t#{sep}"), :type => :array, :into => :stream do |line|
+      line.partition(sep).last
+    end
+  end
+  
   def self.sort_genomic_locations_strict(stream, sep = ":")
     sort_stream(stream, '#', "-k1,1V -k2,2n -t#{sep}")
   end
   
-  def self.sort_genomic_locations(stream)
+  def self.sort_genomic_locations(stream, sep = ":")
     sort_stream(stream, '#', "-k1,1 -k2,2n -t#{sep}")
   end
 
@@ -386,6 +398,7 @@ module Misc
       cmp
     end
   end
+
   def self.intersect_streams_cmp_chr(chr1, chr2)
     chr1 <=> chr2
   end
@@ -492,4 +505,50 @@ module Misc
       end
     end
   end
+
+  def self.genomic_mutations_to_BED(mutations, chr_prefix = false, sort_order = :normal)
+    io = if Array === sort_order
+
+           case chr_prefix.to_s.downcase
+           when "remove"
+             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
+           when "true", "add"
+             sort_order = sort_order.collect{|chr| chr.sub('chr', '') }  if sort_order.first.include?('chr') 
+           end
+
+           sort_genomic_locations_by_contig(mutations, sort_order)
+
+         else
+
+           case sort_order.to_s
+           when 'strict'
+             sort_genomic_locations_strict(mutations)
+           else
+             sort_genomic_locations(mutations)
+           end
+
+         end
+
+    TSV.traverse io, :type => :array, :into => :stream do |mutation|
+      chr, pos, mut, *rest = mutation.split(":")
+      size = case mut
+             when nil
+               1
+             when /^\+(.*)/
+               1 + $1.length
+             when /^\-(.*)/
+               $1.length
+             else
+               mut.length
+             end
+
+      case chr_prefix.to_s.downcase
+      when "true", "add"
+        chr = "chr" + chr if ! chr.include?('chr')
+      when "remove"
+        chr = chr.sub("chr", '') if chr.include?('chr')
+      end
+      [chr, pos.to_i - 1, pos.to_i - 1 + size, mutation] * "\t"
+    end
+  end
 end