rbbt-util 5.21.96 → 5.21.97

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 344b1f349d4ccc887c8d880f838de53f58d82db7
-  data.tar.gz: cc0fef8d5aa0732de3e65b912523696597701efd
+  metadata.gz: 10ef32da69366f0d5db85823d95e5e2461817656
+  data.tar.gz: 7db675c42bfb34aae38611247d22cc5c3c6e4572
 SHA512:
-  metadata.gz: 53fc4f324fa46d7841b64fc47db959f58a9c87202de32a840e91a4ee846e124b56496fe3c3bab4e2e3d8b3ec31032ff3fe87668db6d9393b1a709459ff7db59b
-  data.tar.gz: 9ee1e9a13ca24eb82efda2b69bdf6e3a6d6b806ad3ae49b156788c316d4e6a5a0cb6050029b234db465ae8b6c9347217379fb5df17759844999cf30c29c02969
+  metadata.gz: db31f57eaa344bbbcac12c5a85dad2e46b555f6d19545a55854017c6dc8147d8d321d84f0a2578c29104b1d70b6de322ff5d49dd44c01452fb36fb2ebb59aa0b
+  data.tar.gz: 36f1dbc9bdf5242b02ce9944ce5ae0b93b7b51df4a1a2c68a6370498f7571ca2a9523b5867ce6a637367d26c431d4738a0159de17285f34310a9b0a29d62107a

data/lib/rbbt/tsv/attach/util.rb

@@ -85,7 +85,7 @@ module TSV
                   [source_key]
                 else
                   if other.include? source_key
-                    v = other[source_key][pos]
+                    v = other.type == :flat ?  other[source_key] : other[source_key][pos]
                     Array === v ? v : [v]
                   else
                     [nil]

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -276,6 +276,9 @@ module TSV
         traverse_obj(obj.stream, options, &block)
       when (defined? Step and Step)
 
+        obj.clean if obj.aborted? or obj.recoverable_error?
+        obj.run(true) unless obj.done? or obj.started?
+
         stream = obj.get_stream
 
         if stream

data/lib/rbbt/tsv/parser.rb

@@ -639,6 +639,11 @@ module TSV
       TSV.identify_field(key_field, fields, field)
     end
 
+    def rewind
+      stream.reopen(filename, "r") if stream.closed? and filename
+      stream.rewind
+    end
+
     def self.traverse(stream, options = {}, &block)
       parser = Parser.new(stream, options)
       parser.traverse(options, &block)

data/lib/rbbt/tsv/util.rb

@@ -84,7 +84,7 @@ module TSV
     when (defined? Bgzf and Bgzf)
       file
     when TSV
-      file
+      file.dumper_stream
     when TSV::Dumper
       file.stream
     when TSV::Parser

data/lib/rbbt/util/R.rb

@@ -23,6 +23,7 @@ source('#{UTIL}');
 
     require_sources  = source.collect{|source|
       source = R::LIB_DIR["plot.R"] if source == :plot
+      source = R::LIB_DIR["svg.R"] if source == :svg
       "source('#{source}')"
     } * ";\n" if Array === source and source.any?
 
@@ -64,7 +65,7 @@ source('#{UTIL}');
           f.puts 
           f.puts script
         end
-        CMD.cmd("env R_PROFILE='#{init_file}' xterm \"$RHOME/bin/R\"")
+        CMD.cmd("env R_PROFILE='#{init_file}' xterm \"$R_HOME/bin/R\"")
     end
   end
 

data/lib/rbbt/util/R/plot.rb

@@ -7,7 +7,7 @@ module R
 
       options = options.dup
 
-      sources = [:plot, Rbbt.share.Rlib["svg.R"].find(:lib), options[:source]].flatten.compact
+      sources = [:plot, :svg, options[:source]].flatten.compact
       options.delete :source
 
       fast = options[:fast]

data/lib/rbbt/util/log/progress/report.rb

@@ -165,7 +165,7 @@ module Log
         ellapsed = 0
       end
       ellapsed = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
-      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " in " << Log.color(:green, ellapsed)
+      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed)
       @last_count = 0
       @last_time = @start
       done_msg << " (" << thr_msg << ")"

data/lib/rbbt/util/misc/format.rb

@@ -165,7 +165,8 @@ module Misc
     (string.respond_to? :valid_encoding and string.valid_encoding)
 
     if string.respond_to?(:encode)
-      string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
+      #string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
+      string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
     else
       require 'iconv'
       @@ic ||= Iconv.new('UTF-8//IGNORE', 'UTF-8')

data/lib/rbbt/util/misc/math.rb

@@ -43,7 +43,7 @@ module Misc
   end
 
   def self.mean(list)
-    sum(list) / list.compact.length
+    sum(list).to_f / list.compact.length
   end
 
   def self.median(array)
@@ -52,22 +52,27 @@ module Misc
     (sorted[(len - 1) / 2] + sorted[len / 2]) / 2.0
   end
 
-
-  def self.sd(list)
+  def self.variance(list)
     return nil if list.length < 3
     mean = mean(list)
     list = list.compact
     list_length = list.length
 
-    total_square_distance = 0
+    total_square_distance = 0.0
     list.each do |value|
-      distance = value - mean
+      distance = value.to_f - mean
       total_square_distance += distance * distance
     end
 
     variance = total_square_distance / (list_length - 1)
+
+  end
+
+
+  def self.sd(list)
+    return nil if list.length < 3
+    variance = self.variance(list)
     Math.sqrt(variance)
-    #Math.sqrt(list.compact.inject(0.0){|acc,e| d = e - mean; acc += d * d; acc}) / (list.compact.length - 1)
   end
 
   def self.counts(array)
@@ -124,4 +129,10 @@ module Misc
   def self.in_delta?(a, b, delta = 0.0001)
     (a.to_f - b.to_f).abs < delta
   end
+
+  def self.zscore(e, list)
+    m = Misc.mean(list)
+    sd = Misc.sd(list)
+    (e.to_f - m) / sd
+  end
 end

data/lib/rbbt/util/open.rb

@@ -311,14 +311,17 @@ module Open
   end
 
   def self.gzip?(file)
+    file = file.find if Path === file
     !! (file =~ /\.gz$/)
   end
 
   def self.bgzip?(file)
+    file = file.find if Path === file
     !! (file =~ /\.bgz$/)
   end
 
   def self.zip?(file)
+    file = file.find if Path === file
     !! (file =~ /\.zip$/)
   end
 

data/lib/rbbt/util/simpleopt/parse.rb

@@ -4,6 +4,9 @@ module SOPT
   def self.fix_shortcut(short, long)
     return short unless short and shortcuts.include?(short)
 
+    current = shortcuts.select{|s,l| l == long}.collect{|s,l| s }.first
+    return current if current
+
     chars = long.chars.to_a
     current = [chars.shift]
     short = current * ""

data/lib/rbbt/workflow/accessor.rb

@@ -774,7 +774,8 @@ module Workflow
                      new_=[]
                      dep.each{|d| 
                        if Hash === d
-                         inputs = assign_dep_inputs({}, options.merge(d[:inputs]), real_dependencies, d[:workflow].task_info(d[:task])) 
+                         d[:workflow] ||= self 
+                         inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, d[:workflow].task_info(d[:task])) 
                          d = d[:workflow].job(d[:task], d[:jobname], inputs) 
                        end
                        ComputeDependency.setup(d, compute) if compute

data/lib/rbbt/workflow/step/dependencies.rb

@@ -6,11 +6,6 @@ class Step
   def self.purge_stream_cache
     Log.debug "Purging dup. stream cache"
     STREAM_CACHE_MUTEX.synchronize do
-      #STREAM_CACHE.collect{|k,s| 
-      #  Thread.new do
-      #    Misc.consume_stream s
-      #  end
-      #}
       STREAM_CACHE.clear
     end
   end
@@ -43,17 +38,18 @@ class Step
           new
         end
       end
-    when TSV::Dumper#, TSV::Parser
+    when TSV::Dumper, TSV::Parser
+      orig_stream = stream
       stream = stream.stream
       return stream if stream.closed?
 
       STREAM_CACHE_MUTEX.synchronize do
         if STREAM_CACHE[stream].nil?
-          Log.high "Not duplicating dumper #{ stream.inspect }"
+          Log.high "Not duplicating #{Misc.fingerprint orig_stream} #{ stream.inspect }"
           STREAM_CACHE[stream] = stream
         else
           new = Misc.dup_stream(STREAM_CACHE[stream])
-          Log.high "Duplicating dumper #{ stream.inspect } into #{new.inspect}"
+          Log.high "Duplicating #{Misc.fingerprint orig_stream} #{ stream.inspect } into #{new.inspect}"
           new
         end
       end
@@ -64,6 +60,7 @@ class Step
 
   def dup_inputs
     return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
+    return if ComputeDependency === self and self.compute == :produce
     Log.low "Dupping inputs for #{path}"
     dupped_inputs = @inputs.collect do |input|
       Step.dup_stream input

data/lib/rbbt/workflow/step/run.rb

@@ -51,8 +51,22 @@ class Step
     @inputs.replace new_inputs if step
   end
 
+  def rewind_inputs
+    return if @inputs.nil?
+    Log.debug "Rewinding inputs for #{path}"
+    @inputs.each do |input|
+      next unless input.respond_to? :rewind
+      begin
+        input.rewind
+        Log.debug "Rewinded #{Misc.fingerprint input}"
+      rescue
+      end
+    end
+  end
+
   def _exec
     resolve_input_steps
+    rewind_inputs
     @exec = true if @exec.nil?
     begin
       old = Signal.trap("INT"){ Thread.current.raise Aborted }

data/lib/rbbt/workflow/usage.rb

@@ -3,7 +3,7 @@ require 'rbbt/util/simpleopt'
 module Task
   def doc(workflow = nil, deps = nil)
     puts Log.color(:yellow, "## #{ name }") << ":"
-    puts "\n" << description  << "\n" if description and not description.empty?
+    puts "\n" << Misc.format_paragraph(description.strip)  << "\n" if description and not description.empty?
     puts
 
     case

data/share/Rlib/svg.R

@@ -1,11 +1,11 @@
-library(plyr)
-library(proto)
-library(ggplot2)
-library(gridSVG)
-library(grid)
-library(XML)
-library(ggthemes)
-library(Cairo)
+rbbt.require('plyr')
+rbbt.require('proto')
+rbbt.require('ggplot2')
+rbbt.require('gridSVG')
+rbbt.require('grid')
+rbbt.require('XML')
+rbbt.require('ggthemes')
+rbbt.require('Cairo')
 
 # Modified from http://aaronecay.com/blog/2014/02/tooltips-in-ggplot/
 

data/share/Rlib/util.R

@@ -636,12 +636,13 @@ rbbt.plot.text_scatter <- function(formula, data) {
 
 rbbt.install.CRAN <- function(pkg){
     res = FALSE
-    tryCatch({ install.packages(pkg); res = TRUE }, error = function(e){ warning(paste("Could not install CRAN ", pkg)); res = FALSE })
+    tryCatch({ install.packages(pkg); library(pkg); res = TRUE }, error = function(e){ warning(paste("Could not install CRAN ", pkg)); res = FALSE })
     return(res)
 }
+
 rbbt.install.bioc <-function(pkg){
     res = FALSE
-    tryCatch({ source("http://bioconductor.org/biocLite.R"); biocLite(pkg); res = TRUE }, error = function(e){ warning(paste("Could not install Bioconductor ", pkg)); res = FALSE })
+    tryCatch({ source("http://bioconductor.org/biocLite.R"); biocLite(pkg, ask=FALSE, suppressUpdates = TRUE); res = TRUE }, error = function(e){ warning(paste("Could not install Bioconductor ", pkg)); res = FALSE })
     return(res)
 }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.96
+  version: 5.21.97
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-08 00:00:00.000000000 Z
+date: 2017-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake