rbbt-util 5.27.6 → 5.27.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe7c3dec4d2333e77ea396a2d6e572512bf7033999ff4715f1a1c6c9d3d1b9d5
-  data.tar.gz: 32a3d905c1a95f9c7e5cb359855f8781509cfeecdd8db7b77ee29eaea528d030
+  metadata.gz: be6c76069ffd5a6a49876a42c95f42e7c63975e287446d786c7e4db154456790
+  data.tar.gz: c6bdbb33d38b424dc6355e6dc70a15a4bfefc65b7331d2eb471476b8522ac3a3
 SHA512:
-  metadata.gz: d009024642898f775d423e1ddede6aea538e8ed5ff4675eedadd64768e67b0a46f6519038378a221db76b0e196cfc713ca8dc7553788cdb9a51014c14a97de4c
-  data.tar.gz: 5c0612feca84b00e6d402f7a68ca1cebd6f4b7a785b40899bdaed3385a3f68853824dcc98d9cfd2e641768ef5ca3d52e38e41bdbc1d7a3751ae288e8f8d7ed2d
+  metadata.gz: 1bccd1104a3b2e6236073ca6cd267dba65c0b51a16a6af87c0ea09e1361b8eaf7a9b1a6c7d9bb7e35da3ab4a235ebb578ccd9ce64fd1cdf562282d81a78a3edf
+  data.tar.gz: edf6d4339a6ddafbc0e88d8356686fe7986ae5983ff4a7905be45f419b8d4887b0ed44332f4a537da6cf3e8a3909ce6c687dd0745c70485ff00a7231f9ee0062

data/lib/rbbt/entity.rb

@@ -261,7 +261,7 @@ module Entity
 
         define_method method_name do |*args|
           id = self.id
-          persist_name = orig_method_name.to_s << ":" << (Array === id ? Misc.obj2digest(id) : id)
+          persist_name = orig_method_name.to_s + ":" << (Array === id ? Misc.obj2digest(id) : id)
 
           persist_options = options
           persist_options = persist_options.merge(:other => {:args => args}) if args and args.any?

data/lib/rbbt/fix_width_table.rb

@@ -28,7 +28,7 @@ class FixWidthTable
     else
       Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
       if in_memory
-        @file        = Open.open(@filename, :mode => 'r:ASCII-ASCII'){|f| StringIO.new f.read}
+        @file        = Open.open(@filename, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
       else
         @file        = File.open(@filename, 'r:ASCII-8BIT')
       end

data/lib/rbbt/persist.rb

@@ -365,12 +365,12 @@ module Persist
     type ||= :marshal
 
     persist_options ||= {}
-    if type == :memory and persist_options[:file] and persist_options[:persist] 
+    if type == :memory && persist_options[:file] && persist_options[:persist] 
       repo = persist_options[:repo] || Persist::MEMORY
       if persist_options[:persist] == :update || persist_options[:update]
         repo.delete persist_options[:file]
       end
-      return repo[persist_options[:file]] ||= yield 
+      return repo[persist_options[:file]] ||= yield
     end
 
     if FalseClass === persist_options[:persist]

data/lib/rbbt/tsv/attach.rb

@@ -32,7 +32,8 @@ module TSV
     done = false
     Open.write(output) do |os|
       options.delete :sep if options[:sep] == "\t"
-      os.puts TSV.header_lines(key_field, fields, options) 
+      header_lines = TSV.header_lines(key_field, fields, options) 
+      os.puts header_lines unless header_lines.empty?
 
       while line
         key, *parts = line.sub("\n",'').split(sep, -1)
@@ -212,6 +213,35 @@ module TSV
     other_filename = other.respond_to?(:filename) ? other.filename : other.inspect
     Log.low("Attaching fields:#{Misc.fingerprint fields } from #{other_filename}.")
 
+    same_key = true
+    begin
+      case
+      when (Misc.match_fields(key_field, other.key_field) and same_key)
+        Log.debug "Attachment with same key: #{other.key_field}"
+        attach_same_key other, fields
+      when (not in_namespace and self.fields.select{|f| Misc.match_fields(f, other.key_field)}.any?)
+        Log.debug "Found other key field: #{other.key_field}"
+        attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
+      when (in_namespace and self.fields_in_namespace.select{|f| Misc.match_fields(f, other.key_field)}.any?)
+        Log.debug "Found other key field in #{in_namespace}: #{other.key_field}"
+        attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
+      else
+        index = TSV.find_traversal(self, other, options)
+        raise FieldNotFoundError, "Cannot traverse identifiers" if index.nil?
+        Log.debug "Attachment with index: #{other.key_field}"
+        attach_index other, index, fields
+      end
+    rescue Exception
+      if same_key
+        Log.warn "Could not translate identifiers with same_key"
+        same_key = false
+        retry
+      else
+        raise $!
+      end
+    end
+    Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
+
     if complete
       fill = TrueClass === complete ? nil : complete
       field_length = self.fields.length 
@@ -219,6 +249,9 @@ module TSV
       other_common_pos = common_fields.collect{|f| other.fields.index f}
       this_common_pos = common_fields.collect{|f| self.fields.index f}
       missing = other.keys - self.keys
+
+      other = other.to_list if other.type == :single
+
       case type
       when :single
         missing.each do |k|
@@ -249,35 +282,6 @@ module TSV
       end
     end
 
-    same_key = true
-    begin
-      case
-      when (Misc.match_fields(key_field, other.key_field) and same_key)
-        Log.debug "Attachment with same key: #{other.key_field}"
-        attach_same_key other, fields
-      when (not in_namespace and self.fields.select{|f| Misc.match_fields(f, other.key_field)}.any?)
-        Log.debug "Found other key field: #{other.key_field}"
-        attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
-      when (in_namespace and self.fields_in_namespace.select{|f| Misc.match_fields(f, other.key_field)}.any?)
-        Log.debug "Found other key field in #{in_namespace}: #{other.key_field}"
-        attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
-      else
-        index = TSV.find_traversal(self, other, options)
-        raise FieldNotFoundError, "Cannot traverse identifiers" if index.nil?
-        Log.debug "Attachment with index: #{other.key_field}"
-        attach_index other, index, fields
-      end
-    rescue Exception
-      if same_key
-        Log.warn "Could not translate identifiers with same_key"
-        same_key = false
-        retry
-      else
-        raise $!
-      end
-    end
-    Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
-
     self
   end
 

data/lib/rbbt/tsv/attach/util.rb

@@ -4,9 +4,20 @@ module TSV
     fields = other.fields - [key_field].concat(self.fields) if fields.nil?
 
     fields = [fields].compact unless Array === fields
+
+    common_fields = self.fields & fields
+
+    fields = fields - common_fields
+
     num_fields = fields.length
 
     field_positions = fields.collect{|field| other.identify_field field}
+
+    if common_fields.any?
+      common_field_positions = common_fields.collect{|field| self.identify_field field}
+      common_field_positions_other = common_fields.collect{|field| other.identify_field field}
+    end
+
     other.with_unnamed do
       with_unnamed do
         through do |key, values|
@@ -28,6 +39,12 @@ module TSV
               new_values = field_positions.collect do |pos|
                 pos == :key ? key : other_values[pos]
               end
+
+              if common_fields.any?
+                common_field_positions.zip(common_field_positions_other).each do |p1,p2|
+                  current[p1] += other_values[p2]
+                end
+              end
             end
 
             new_values.collect!{|v| [v]}     if     type == :double and not (other.type == :double or other.type == :flat)

data/lib/rbbt/tsv/stream.rb

@@ -89,6 +89,7 @@ module TSV
         stream
       end
 
+      all_fields = fields
       key_field = key_fields.compact.first
       if same_fields
         fields = fields.first

data/lib/rbbt/util/misc/development.rb

@@ -292,14 +292,16 @@ def self.add_libdir(dir=nil)
     end
   end
 
-  def self.bootstrap(elems, num = :current, options = {}, &block)
+  def self.bootstrap(elems, num = nil, options = {}, &block)
     IndiferentHash.setup options
+
+    num = Rbbt::Config.get :cpus, :default_bootstrap_cpus, :bootstrap_cpus if num == :current || num == nil
     num = :current if num.nil?
     cpus = case num
            when :current
-            n = 10
-            n = elems.length / 2 if n > elems.length/2
-            n
+             n = Etc.nprocessors
+             n = elems.length / 2 if n > elems.length/2
+             n
            when String
              num.to_i
            when Integer

data/lib/rbbt/util/misc/format.rb

@@ -163,7 +163,7 @@ module Misc
 
   def self.fixutf8(string)
     return nil if string.nil?
-    return string if string.respond_to?(:encoding) && string.encoding == "UTF-8" && (string.respond_to?(:valid_encoding?) && string.valid_encoding?) ||
+    return string if string.respond_to?(:encoding) && string.encoding.to_s == "UTF-8" && (string.respond_to?(:valid_encoding?) && string.valid_encoding?) ||
                      (string.respond_to?(:valid_encoding) && string.valid_encoding)
 
     if string.respond_to?(:encode)

data/lib/rbbt/util/misc/pipes.rb

@@ -18,6 +18,8 @@ module Misc
 
   BLOCK_SIZE=1024 * 8
 
+  SKIP_TAG="[SKIP TAG]"
+
   PIPE_MUTEX = Mutex.new
 
   OPEN_PIPE_IN = []
@@ -29,7 +31,7 @@ module Misc
 
       [sout, sin]
     end
-    Log.debug{"Creating pipe #{[res.last.inspect,res.first.inspect] * " => "}"}
+    Log.debug{"Creating pipe #{[res.last.inspect, res.first.inspect] * " => "}"}
     res
   end
 
@@ -255,6 +257,11 @@ module Misc
     end
     tee1, *rest = Misc.tee_stream stream_dup, num + 1
     stream.reopen(tee1)
+
+    #ToDo: I can't explain why the @threads variable appears with the value of
+    # @filename
+    stream.instance_variable_set(:@threads, nil) if stream.instance_variables.include?(:@threads)
+
     tee1.annotate(stream)
     rest
   end
@@ -537,18 +544,29 @@ module Misc
     end
   end
 
+  def self.buffer_stream(stream)
+    sout, sin = Misc.pipe
+    Misc.consume_stream(stream, true, sin)
+    sout
+  end
+
   def self._paste_streams(streams, output, lines = nil, sep = "\t", header = nil, &block)
     output.puts header if header
     streams = streams.collect do |stream|
       if defined? Step and Step === stream
-        stream.get_stream || stream.join.path.open
+        io = stream.get_stream 
+        if io
+          buffer_stream(io)
+        else
+          stream.join.path.open
+        end
       else
         stream
       end
     end
 
     begin
-      done_streams = []
+
       lines ||= streams.collect{|s| s.gets }
       keys = []
       parts = []
@@ -564,6 +582,7 @@ module Misc
       end
       sizes = parts.collect{|p| p.nil? ? 0 : p.length }
       last_min = nil
+
       while lines.compact.any?
         if block_given?
           min = keys.compact.sort(&block).first
@@ -571,14 +590,23 @@ module Misc
           min = keys.compact.sort.first
         end
         str = []
+        threads = []
         keys.each_with_index do |key,i|
           case key
           when min
-            str << [parts[i] * sep]
+            if parts[i] == [SKIP_TAG]
+              str << [sep * (sizes[i]-1)] if sizes[i] > 0
+            else
+              str << [parts[i] * sep]
+            end
+
             line = lines[i] = streams[i].gets
-            if line.nil?
+
+            if line.nil? 
               keys[i] = nil
               parts[i] = nil
+              streams[i].close unless streams[i].closed?
+              streams[i].join if streams[i].respond_to?(:join) 
             else
               k, *p = line.chomp.split(sep, -1)
               keys[i] = k
@@ -589,10 +617,12 @@ module Misc
           end
         end
 
-        output.puts [min, str*sep] * sep
+        output.puts [min, str.flatten*sep] * sep
       end
+
       streams.each do |stream|
-        stream.join if stream.respond_to? :join
+        stream.close unless stream.closed?
+        stream.join if stream.respond_to?(:join) 
       end
     rescue 
       Log.exception $!

data/lib/rbbt/util/tmpfile.rb

@@ -17,14 +17,14 @@ module TmpFile
 
   # Creates a random file name, with the given suffix and a random number
   # up to +max+
-  def self.random_name(s = "tmp-", max = 10000000)
+  def self.random_name(s = "tmp-", max = 1_000_000_000)
     n = rand(max)
     s + n.to_s
   end
 
   # Creates a random filename in the temporary directory
-  def self.tmp_file(s = "tmp-", max=10000000, dir = TMPDIR)
-    File.expand_path(File.join(dir, random_name(s,max)))
+  def self.tmp_file(s = "tmp-", max=1_000_000_000, dir = TMPDIR)
+    File.expand_path(File.join(dir, random_name(s, max)))
   end
 
   def self.with_file(content = nil, erase = true, options = {})
@@ -33,7 +33,7 @@ module TmpFile
 
     prefix = options[:prefix] || "tmp-"
     tmpdir = options[:tmpdir] || TMPDIR
-    max = options[:max] || 10000000
+    max = options[:max] || 1_000_000_000
     tmpfile = tmp_file prefix, max, tmpdir
     if options[:extension]
       tmpfile += ".#{options[:extension]}"

data/lib/rbbt/workflow.rb

@@ -10,9 +10,6 @@ require 'rbbt/workflow/util/provenance'
 
 module Workflow
 
-  STEP_CACHE = {}
-  LOAD_STEP_CACHE = {}
-
   class TaskNotFoundException < Exception 
     def initialize(workflow, task = nil)
       if task
@@ -264,11 +261,11 @@ module Workflow
   end
 
   def step_cache
-    @step_cache ||= Workflow::STEP_CACHE
+    Thread.current[:step_cache] ||= {}
   end
 
   def self.load_step_cache
-    @load_step_cache ||= Workflow::LOAD_STEP_CACHE
+    Thread.current[:load_step_cache] ||= {}
   end
 
 

data/lib/rbbt/workflow/accessor.rb

@@ -418,7 +418,7 @@ module Workflow
                        when :hash
                          clean_inputs = Annotated.purge(inputs)
                          clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
-                         deps_str = dependencies.collect{|d| Step === d ? d.short_path : d }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << d.short_path : d }
                          key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
                          key_str = Misc.obj2str(key_obj)
                          hash_str = Misc.digest(key_str)

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -137,14 +137,16 @@ class RemoteWorkflow
 
         post_thread = Thread.new(Thread.current) do |parent|
           bl = lambda do |rok|
-            if Net::HTTPOK === rok 
+            case rok
+            when Net::HTTPOK
               _url = rok["RBBT-STREAMING-JOB-URL"]
               @url = File.join(task_url, File.basename(_url)) if _url
               rok.read_body do |c,_a, _b|
                 sin.write c
               end
               sin.close
-            elsif Net::HTTPSeeOther === rok
+            when Net::HTTPRedirection, Net::HTTPAccepted
+              Thread.current.report_on_exception = false
               raise TryThis.new(rok)
             else
               err = StringIO.new
@@ -178,7 +180,8 @@ class RemoteWorkflow
           begin
             RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
           rescue TryThis
-            RestClient::Request.execute(:method => :get, :url => $!.payload.header[:location], :block_response => bl)
+            url = $!.payload["location"]
+            RestClient::Request.execute(:method => :get, :url => url, :block_response => bl)
           end
         end
 

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -26,8 +26,8 @@ class RemoteStep < Step
 
   def cache_file
     begin
-      digest = Misc.obj2digest([base_url, task, base_name, inputs])
-      Rbbt.var.cache.REST[[task, clean_name, digest].compact * "."].find
+      digest = Misc.obj2digest([base_url, task.to_s, base_name, inputs])
+      Rbbt.var.cache.REST[task.to_s][[clean_name, digest].compact * "."].find
     rescue
       Log.exception $!
       raise $!
@@ -143,7 +143,11 @@ class RemoteStep < Step
     @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => update) do
       @last_info_time = Time.now
       init_job unless @url
-      info = @adaptor.get_json(File.join(@url, 'info'))
+      info = begin
+               @adaptor.get_json(File.join(@url, 'info'))
+             rescue
+               {:status => :noinfo}
+             end
       info = RemoteWorkflow.fix_hash(info)
       info[:status] = info[:status].to_sym if String === info[:status]
       info
@@ -243,6 +247,7 @@ class RemoteStep < Step
   def join
     return true if cache_files.any?
     init_job unless @url
+    produce unless @started
     Log.debug{ "Joining RemoteStep: #{path}" }
 
     if IO === @result
@@ -258,7 +263,6 @@ class RemoteStep < Step
       sleep 1 unless self.done? || self.aborted? || self.error?
       while not (self.done? || self.aborted? || self.error?)
         sleep 3
-        iif [self.done?, self.status, self.info]
       end
     end
 
@@ -311,6 +315,10 @@ class RemoteStep < Step
     end
   end
 
+  def short_path
+    init_job unless @url
+    [@task, @name] * "/"
+  end
 
   def input_checks
     []

data/lib/rbbt/workflow/step/dependencies.rb

@@ -366,7 +366,7 @@ class Step
       (step.dependencies + step.input_dependencies).each do |step_dep|
         next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
-        dep_step[step_dep.path] << step_dep
+        dep_step[step_dep.path] << step
       end
     end
 

data/lib/rbbt/workflow/usage.rb

@@ -88,7 +88,7 @@ module Workflow
                           key = [workflow, task]
 
                           dep_tree[key] = workflow.dep_tree(task)
-                        end if self.task_dependencies[name.to_sym]
+                        end if name && self.task_dependencies[name.to_sym]
                         dep_tree
                       end
   end

data/share/rbbt_commands/app/start

@@ -84,6 +84,10 @@ Misc.in_dir(app_dir) do
     end
   end
 
+  fixed_options = {}
+  options.each do |k,v| fixed_options[k.to_sym] = v end
+  options = fixed_options
+
   case server
   when 'passenger'
     system ENV, "env RBBT_LOG=0 passenger start -R '#{config_ru_file}' -p #{options[:Port] || "2887"}"

data/share/rbbt_commands/system/clean

@@ -97,7 +97,7 @@ TSV.traverse jobs do |file,i|
   else
     info = begin
              Open.open(i[:info_file]) do |f|
-               Step::INFO_SERIALIAZER.load(f)
+               Step::INFO_SERIALIZER.load(f)
              end
            rescue
              {:status => :noinfo}
@@ -134,7 +134,7 @@ TSV.traverse jobs do |file,i|
     end
   end
 
-  if (force and status !~ /done/) or 
+  if (force && status !~ /done/) or 
     status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync)$/ or
     (status == "noinfo" and not done) or
     status == "" 

data/share/rbbt_commands/tsv/uncollapse

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+options = SOPT.setup <<EOF
+Read a TSV file and dump it
+
+$ rbbt tsv read [options] <filename.tsv|->
+
+Use - to read from STDIN
+
+-k--key_field* Key field
+-f--fields* Fields
+-t--type* Type
+-m--merge* Merge from multiple rows
+-h--help Print this help
+
+EOF
+rbbt_usage and exit 0 if options[:help]
+
+file = ARGV.shift
+
+file = STDIN if file == '-'
+
+tsv = TSV.open file, :merge => true, :type => :double
+
+
+puts tsv.to_unmerged_expanded_s

data/share/rbbt_commands/workflow/info

@@ -153,7 +153,7 @@ if recursive
     dep = deps.shift
     inputs = {} if inputs.nil?
     inputs = inputs.merge(dep.info[:inputs] || {})
-    deps = deps.concat dep.dependencies
+    deps = deps.concat dep.dependencies if dep.dependencies
   end
 
   inputs = inputs.merge step.archived_inputs

data/share/rbbt_commands/workflow/server

@@ -123,15 +123,11 @@ TmpFile.with_file do |app_dir|
     else
       options[:config] = config_ru_file
 
-			clean_options = {}
-			options.each do |k,v|
-        begin
-          clean_options[k.to_sym] = v
-        rescue
-        end
-      end
+      fixed_options = {}
+      options.each do |k,v| fixed_options[k.to_sym] = v end
+      options = fixed_options
 
-      Rack::Server.start(clean_options)
+      Rack::Server.start(options)
     end
   end
 end

data/share/rbbt_commands/workflow/task

@@ -337,6 +337,8 @@ if clean_task
       dep.clean 
       dep.set_info :status, :cleaned
     end
+
+    job.clean if job.task_name.to_s == clean_task.to_s
   end
 end
 

data/test/rbbt/tsv/test_attach.rb

@@ -213,8 +213,9 @@ row2    E
 
     tsv1 = tsv2 = nil
 
-    tsv1 = Rbbt.tmp.test.test1.data.tsv :double,  :sep => /\s+/
-    tsv2 = Rbbt.tmp.test.test2.data.tsv :double,  :sep => /\s+/
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+    Log.tsv tsv2
 
     tsv2.identifiers = Rbbt.tmp.test.test2.identifiers.produce.find #.to_s
 

data/test/rbbt/tsv/test_stream.rb

@@ -4,6 +4,7 @@ require 'rbbt/tsv/stream'
 require 'rbbt'
 
 class TestStream < Test::Unit::TestCase
+
   def test_collapse_stream
     text=<<-EOF
 #: :sep=" "
@@ -65,7 +66,7 @@ row3 AAA BBB CCC
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row1 a b
 row3 aaa bbb
 row2 aa bb
@@ -100,7 +101,7 @@ row1 A B C
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row2 aa bb
     EOF
 
@@ -132,7 +133,7 @@ row1 A B C
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row2 aa bb
     EOF
 

data/test/rbbt/util/misc/test_format.rb

@@ -0,0 +1,10 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/misc/format'
+
+class TestClass < Test::Unit::TestCase
+  def test_fixutf8
+    a = "Camión"
+    assert_equal a, Misc.fixutf8(a)
+  end
+end
+

data/test/rbbt/util/misc/test_pipes.rb

@@ -285,7 +285,7 @@ line4
       begin
         Misc.consume_stream(sio, false, STDOUT)
       rescue
-        Log.exception $!
+        raise $!
       end
     end
   end

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -4,6 +4,27 @@ require 'rbbt/workflow'
 module DepWorkflow
   extend Workflow
 
+  input :input_file, :file, "Input file", nil, :stream => true
+  task :s1 => :array do |input_file|
+    TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
+      line + "\t" << "Task1"
+    end
+  end
+
+  dep :s1
+  task :s2 => :array do |input_file|
+    TSV.traverse step(:s1), :type => :array, :into => :stream, :bar => "Task2" do |line|
+      next [line.split("\t").first, Misc::SKIP_TAG] * "\t" if rand < 0.9
+      line + "\t" << "Task2"
+    end
+  end
+
+  dep :s1
+  dep :s2
+  task :s3 => :array do |input_file|
+    Misc.paste_streams(dependencies.reverse)
+  end
+
   input :input_file, :file, "Input file", nil, :stream => true
   task :task1 => :array do |input_file|
     TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
@@ -28,9 +49,7 @@ module DepWorkflow
   dep :task2
   dep :task3
   task :task4 => :array do
-    s1 = TSV.get_stream step(:task2)
-    s2 = TSV.get_stream step(:task3)
-    Misc.paste_streams([s1, s2])
+    Misc.paste_streams(dependencies)
   end
 
   dep :task4
@@ -40,12 +59,10 @@ module DepWorkflow
     end
   end
 
-  dep :task5
   dep :task2
+  dep :task5
   task :task6 => :array do
-    s1 = TSV.get_stream step(:task2)
-    s2 = TSV.get_stream step(:task5)
-    Misc.paste_streams([s1, s2])
+    Misc.paste_streams(dependencies)
   end
 
   input :stream_file, :file, "Streamed file", nil, :stream => true
@@ -134,8 +151,25 @@ class TestWorkflowDependency < Test::Unit::TestCase
     end
   end
 
+  def test_task3
+    size = 100000
+    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    TmpFile.with_file(content) do |input_file|
+      job = DepWorkflow.job(:task3, "TEST", :input_file => input_file)
+      io = TSV.get_stream job.run(:stream)
+      last_line = nil
+      while line = io.gets
+        last_line = line.strip
+      end
+      io.join
+
+      assert_equal "Line #{size}\tTask1\tTask3", last_line
+    end
+  end
+
   def test_task4
-    size = 1000
+    size = 100000
+    Log.severity = 0
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     TmpFile.with_file(content) do |input_file|
@@ -151,7 +185,7 @@ class TestWorkflowDependency < Test::Unit::TestCase
   end
   
   def test_task5
-    size = 1000
+    size = 10000
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     TmpFile.with_file(content) do |input_file|
@@ -165,9 +199,32 @@ class TestWorkflowDependency < Test::Unit::TestCase
     assert_equal "Line #{size}\tTask1\tTask2\tTask1\tTask3\tTask5", last_line
   end
   
+  def test_s3
+    size = 100000
+    content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    last_line = nil
+    Log.severity = 0
+    TmpFile.with_file(content) do |input_file|
+      begin
+        job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
+        job.recursive_clean
+        job.run(:stream)
+        io = TSV.get_stream job
+        while line = io.gets
+          last_line = line.strip
+        end
+        io.join if io.respond_to? :join
+      rescue Exception
+        job.abort
+        raise $!
+      end
+    end
+    assert last_line.include? "Line #{size}"
+  end
+  
   def test_task6
     size = 100000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     Log.severity = 0
     TmpFile.with_file(content) do |input_file|
@@ -189,7 +246,7 @@ class TestWorkflowDependency < Test::Unit::TestCase
   end
   
   def test_task8
-    size = 100000
+    size = 10000
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     Log.severity = 0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.27.6
+  version: 5.27.11
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-03 00:00:00.000000000 Z
+date: 2020-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -403,6 +403,7 @@ files:
 - share/rbbt_commands/tsv/sort
 - share/rbbt_commands/tsv/subset
 - share/rbbt_commands/tsv/transpose
+- share/rbbt_commands/tsv/uncollapse
 - share/rbbt_commands/tsv/unzip
 - share/rbbt_commands/tsv/values
 - share/rbbt_commands/tsv/write_excel
@@ -487,6 +488,7 @@ files:
 - test/rbbt/util/concurrency/test_threads.rb
 - test/rbbt/util/log/test_progress.rb
 - test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_omics.rb
@@ -552,6 +554,7 @@ test_files:
 - test/rbbt/util/test_colorize.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_bgzf.rb