rbbt-util 5.27.6 → 5.27.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe7c3dec4d2333e77ea396a2d6e572512bf7033999ff4715f1a1c6c9d3d1b9d5
-  data.tar.gz: 32a3d905c1a95f9c7e5cb359855f8781509cfeecdd8db7b77ee29eaea528d030
+  metadata.gz: 6551d632ae1f2289805a53ceaf9db01d0c0a029ea6c76176aed419e4d6a2a485
+  data.tar.gz: 72562729535554e718451adb87338503ff40a81874a9e0fed6f8eaf4e6aa0edc
 SHA512:
-  metadata.gz: d009024642898f775d423e1ddede6aea538e8ed5ff4675eedadd64768e67b0a46f6519038378a221db76b0e196cfc713ca8dc7553788cdb9a51014c14a97de4c
-  data.tar.gz: 5c0612feca84b00e6d402f7a68ca1cebd6f4b7a785b40899bdaed3385a3f68853824dcc98d9cfd2e641768ef5ca3d52e38e41bdbc1d7a3751ae288e8f8d7ed2d
+  metadata.gz: 0d8c3ff2861dd4bbd835e3e41826f5fd6452bdd0b40fbd8378737effdffa6da051ce221e24f4bbacaf952725ccf7da09b533c8c414a7fd10480369ae528a4259
+  data.tar.gz: 1790b8f38f91dfbb582a394d86953156d90351a8d724fb9535a3bc72ebac5fbbfe24e88fe10cf943e2ed91ff059af68cf581bac333f5480096f85cf6e29b8018

data/lib/rbbt/tsv/stream.rb

@@ -89,6 +89,7 @@ module TSV
         stream
       end
 
+      all_fields = fields
       key_field = key_fields.compact.first
       if same_fields
         fields = fields.first

data/lib/rbbt/util/misc/pipes.rb

@@ -18,6 +18,8 @@ module Misc
 
   BLOCK_SIZE=1024 * 8
 
+  SKIP_TAG="[SKIP TAG]"
+
   PIPE_MUTEX = Mutex.new
 
   OPEN_PIPE_IN = []
@@ -29,7 +31,7 @@ module Misc
 
       [sout, sin]
     end
-    Log.debug{"Creating pipe #{[res.last.inspect,res.first.inspect] * " => "}"}
+    Log.debug{"Creating pipe #{[res.last.inspect, res.first.inspect] * " => "}"}
     res
   end
 
@@ -255,6 +257,11 @@ module Misc
     end
     tee1, *rest = Misc.tee_stream stream_dup, num + 1
     stream.reopen(tee1)
+
+    #ToDo: I can't explain why the @threads variable appears with the value of
+    # @filename
+    stream.instance_variable_set(:@threads, nil) if stream.instance_variables.include?(:@threads)
+
     tee1.annotate(stream)
     rest
   end
@@ -537,18 +544,29 @@ module Misc
     end
   end
 
+  def self.buffer_stream(stream)
+    sout, sin = Misc.pipe
+    Misc.consume_stream(stream, true, sin)
+    sout
+  end
+
   def self._paste_streams(streams, output, lines = nil, sep = "\t", header = nil, &block)
     output.puts header if header
     streams = streams.collect do |stream|
       if defined? Step and Step === stream
-        stream.get_stream || stream.join.path.open
+        io = stream.get_stream 
+        if io
+          buffer_stream(io)
+        else
+          stream.join.path.open
+        end
       else
         stream
       end
     end
 
     begin
-      done_streams = []
+
       lines ||= streams.collect{|s| s.gets }
       keys = []
       parts = []
@@ -564,6 +582,7 @@ module Misc
       end
       sizes = parts.collect{|p| p.nil? ? 0 : p.length }
       last_min = nil
+
       while lines.compact.any?
         if block_given?
           min = keys.compact.sort(&block).first
@@ -571,14 +590,23 @@ module Misc
           min = keys.compact.sort.first
         end
         str = []
+        threads = []
         keys.each_with_index do |key,i|
           case key
           when min
-            str << [parts[i] * sep]
+            if parts[i] == [SKIP_TAG]
+              str << [sep * (sizes[i]-1)] if sizes[i] > 0
+            else
+              str << [parts[i] * sep]
+            end
+
             line = lines[i] = streams[i].gets
-            if line.nil?
+
+            if line.nil? 
               keys[i] = nil
               parts[i] = nil
+              streams[i].close unless streams[i].closed?
+              streams[i].join if streams[i].respond_to?(:join) 
             else
               k, *p = line.chomp.split(sep, -1)
               keys[i] = k
@@ -589,10 +617,12 @@ module Misc
           end
         end
 
-        output.puts [min, str*sep] * sep
+        output.puts [min, str.flatten*sep] * sep
       end
+
       streams.each do |stream|
-        stream.join if stream.respond_to? :join
+        stream.close unless stream.closed?
+        stream.join if stream.respond_to?(:join) 
       end
     rescue 
       Log.exception $!

data/lib/rbbt/workflow/step/dependencies.rb

@@ -366,7 +366,7 @@ class Step
       (step.dependencies + step.input_dependencies).each do |step_dep|
         next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
-        dep_step[step_dep.path] << step_dep
+        dep_step[step_dep.path] << step
       end
     end
 

data/test/rbbt/tsv/test_stream.rb

@@ -4,6 +4,7 @@ require 'rbbt/tsv/stream'
 require 'rbbt'
 
 class TestStream < Test::Unit::TestCase
+
   def test_collapse_stream
     text=<<-EOF
 #: :sep=" "
@@ -65,7 +66,7 @@ row3 AAA BBB CCC
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row1 a b
 row3 aaa bbb
 row2 aa bb
@@ -100,7 +101,7 @@ row1 A B C
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row2 aa bb
     EOF
 
@@ -132,7 +133,7 @@ row1 A B C
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row2 aa bb
     EOF
 

data/test/rbbt/util/misc/test_pipes.rb

@@ -285,7 +285,7 @@ line4
       begin
         Misc.consume_stream(sio, false, STDOUT)
       rescue
-        Log.exception $!
+        raise $!
       end
     end
   end

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -4,6 +4,27 @@ require 'rbbt/workflow'
 module DepWorkflow
   extend Workflow
 
+  input :input_file, :file, "Input file", nil, :stream => true
+  task :s1 => :array do |input_file|
+    TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
+      line + "\t" << "Task1"
+    end
+  end
+
+  dep :s1
+  task :s2 => :array do |input_file|
+    TSV.traverse step(:s1), :type => :array, :into => :stream, :bar => "Task2" do |line|
+      next [line.split("\t").first, Misc::SKIP_TAG] * "\t" if rand < 0.9
+      line + "\t" << "Task2"
+    end
+  end
+
+  dep :s1
+  dep :s2
+  task :s3 => :array do |input_file|
+    Misc.paste_streams(dependencies.reverse)
+  end
+
   input :input_file, :file, "Input file", nil, :stream => true
   task :task1 => :array do |input_file|
     TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
@@ -28,9 +49,7 @@ module DepWorkflow
   dep :task2
   dep :task3
   task :task4 => :array do
-    s1 = TSV.get_stream step(:task2)
-    s2 = TSV.get_stream step(:task3)
-    Misc.paste_streams([s1, s2])
+    Misc.paste_streams(dependencies)
   end
 
   dep :task4
@@ -40,12 +59,10 @@ module DepWorkflow
     end
   end
 
-  dep :task5
   dep :task2
+  dep :task5
   task :task6 => :array do
-    s1 = TSV.get_stream step(:task2)
-    s2 = TSV.get_stream step(:task5)
-    Misc.paste_streams([s1, s2])
+    Misc.paste_streams(dependencies)
   end
 
   input :stream_file, :file, "Streamed file", nil, :stream => true
@@ -134,8 +151,25 @@ class TestWorkflowDependency < Test::Unit::TestCase
     end
   end
 
+  def test_task3
+    size = 100000
+    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    TmpFile.with_file(content) do |input_file|
+      job = DepWorkflow.job(:task3, "TEST", :input_file => input_file)
+      io = TSV.get_stream job.run(:stream)
+      last_line = nil
+      while line = io.gets
+        last_line = line.strip
+      end
+      io.join
+
+      assert_equal "Line #{size}\tTask1\tTask3", last_line
+    end
+  end
+
   def test_task4
-    size = 1000
+    size = 100000
+    Log.severity = 0
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     TmpFile.with_file(content) do |input_file|
@@ -151,7 +185,7 @@ class TestWorkflowDependency < Test::Unit::TestCase
   end
   
   def test_task5
-    size = 1000
+    size = 10000
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     TmpFile.with_file(content) do |input_file|
@@ -165,9 +199,32 @@ class TestWorkflowDependency < Test::Unit::TestCase
     assert_equal "Line #{size}\tTask1\tTask2\tTask1\tTask3\tTask5", last_line
   end
   
+  def test_s3
+    size = 100000
+    content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    last_line = nil
+    Log.severity = 0
+    TmpFile.with_file(content) do |input_file|
+      begin
+        job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
+        job.recursive_clean
+        job.run(:stream)
+        io = TSV.get_stream job
+        while line = io.gets
+          last_line = line.strip
+        end
+        io.join if io.respond_to? :join
+      rescue Exception
+        job.abort
+        raise $!
+      end
+    end
+    assert last_line.include? "Line #{size}"
+  end
+  
   def test_task6
     size = 100000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     Log.severity = 0
     TmpFile.with_file(content) do |input_file|
@@ -189,7 +246,7 @@ class TestWorkflowDependency < Test::Unit::TestCase
   end
   
   def test_task8
-    size = 100000
+    size = 10000
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     Log.severity = 0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.27.6
+  version: 5.27.7
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-03 00:00:00.000000000 Z
+date: 2020-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake