rbbt-util 5.10.1 → 5.10.2

data/lib/rbbt/workflow/definition.rb

@@ -26,6 +26,8 @@ module Workflow
   end
 
   def dep(*dependency_list, &block)
+    @dependency_list ||= []
+    @dependencies ||= []
     dependency_list << block if block_given?
     dependencies.concat dependency_list
   end

data/lib/rbbt/workflow/step.rb

@@ -102,7 +102,8 @@ class Step
   def join
     case @result
     when IO
-      while @result.read 2048; end
+      while @result.read 2048; Thread.pass end unless @result.closed? or @result.eof?
+      @result.join if @result.respond_to? :join
       @result = nil
     end
 
@@ -126,7 +127,7 @@ class Step
 
   def run(no_load = false)
 
-    result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => false do
+    result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load ? :stream : false do
       if Step === Step.log_relay_step and not self == Step.log_relay_step
         relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
       end
@@ -190,12 +191,18 @@ class Step
       end
 
       case result
-      when IO, TSV::Dumper
-        log :streaming, "#{Log.color :magenta, "Streaming task result"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
-        class << result
-          attr_accessor :callback
+      when IO, StringIO
+        log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+        ConcurrentStream.setup result do
+          eee 1
+          set_info :done, (done_time = Time.now)
+          set_info :time_elapsed, (time_elapsed = done_time - start_time)
+          log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
         end
-        result.callback = Proc.new do
+      when TSV::Dumper
+        log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+        ConcurrentStream.setup result.stream do
+          set_info :done, (done_time = Time.now)
           set_info :done, (done_time = Time.now)
           set_info :time_elapsed, (time_elapsed = done_time - start_time)
           log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
@@ -256,6 +263,7 @@ class Step
           exit -1
         end
         set_info :pid, nil
+        exit 0
       ensure
         RbbtSemaphore.post_semaphore(semaphore) if semaphore
       end
@@ -297,7 +305,7 @@ class Step
   end
 
   def load
-    return prepare_result @result, @task.result_description if @result
+    return prepare_result @result, @task.result_description if @result and not @path == @result
     join if not done?
     return Persist.load_file(@path, @task.result_type) if @path.exists?
     exec

data/share/rbbt_commands/benchmark/throughput

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Monitor throughput
+
+$ rbbt tsv throughput 
+
+Display summary information. Works with Tokyocabinet HDB and BDB as well.
+
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+last = start = Time.now
+count = 0
+max = 0
+avg = 0
+all = []
+scale = 5
+while line = STDIN.gets
+  count += 1
+  if Time.now - last >= 1.0 / scale
+    Log.clear_line
+    puts "#{ count*scale } per second. Max #{max*scale}. Average #{avg*scale}"
+    last = Time.now
+    max = count > max ? count : max
+    all << count
+    avg = Misc.mean(all).to_i if all.length > 3
+    count = 0
+  end
+end
+
+all << count
+
+puts "Total #{Misc.sum(all).to_i} in #{(Time.now - start).to_i} seconds -- #{(Misc.sum(all).to_f / (Time.now - start)).to_i } per second. Max #{max*scale}. Average #{avg*scale}"

data/share/rbbt_commands/system/report

@@ -53,20 +53,25 @@ options[:app].split(/,|\s/).collect do |app|
   app_dirs[app] = report
 end if options[:app]
 
-puts <<EOF
 
-#{ report_jobs workflow_dir}
-#{ app_dirs.collect{|d,report| report } * "\n" }
+puts <<EOF
+# LOCKS
+#{ CMD.cmd("find #{Rbbt.share.find_all.collect{|f| "'#{f}'" } * " " } -name '*.lock'").read << CMD.cmd("find #{Rbbt.var.find_all.collect{|f| "'#{f}'" } * " " } -name '*.lock'").read }
+EOF
 
+puts <<EOF
 # LOCKED TSV
 #{ Rbbt.var.tsv_open_locks.glob('*').collect{|f| "- " << File.basename(f) } * "\n" }
+EOF
 
-# LOCKS
-#{ CMD.cmd("find #{Rbbt.share.find_all.collect{|f| "'#{f}'" } * " " } -name '*.lock'").read << CMD.cmd("find #{Rbbt.var.find_all.collect{|f| "'#{f}'" } * " " } -name '*.lock'").read }
-
+puts <<EOF
 # PERSIST
 #{ CMD.cmd("find #{Rbbt.share.find_all.collect{|f| "'#{f}'" } * " " } -name '*.persist'").read << CMD.cmd("find #{Rbbt.var.find_all.collect{|f| "'#{f}'" } * " " } -name '*.persist'").read }
 EOF
 
-
+puts <<EOF
+# JOBS
+#{ report_jobs workflow_dir}
+#{ app_dirs.collect{|d,report| report } * "\n" }
+EOF
 

data/share/rbbt_commands/workflow/task

@@ -404,10 +404,11 @@ when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
 when Step
   if IO === res.result
     io = res.result
-    while line = io.gets do
-      out.puts line
-    end
-    io.close
+    Thread.pass while IO.select([io]).nil?
+    while block = io.read(2048) do
+      out.write block
+    end unless io.closed? 
+    io.join if io.respond_to? :join
   else
     res.join
     out.puts Open.read(res.path) if File.exists? res.path

data/test/rbbt/test_persist.rb

@@ -12,7 +12,7 @@ end
 
 class TestPersist < Test::Unit::TestCase
 
-  def test_array_persist
+  def _test_array_persist
     TmpFile.with_file do |tmp|
       10.times do
         assert_equal ["1", "2"],(Persist.persist("Test", :array, :file => tmp) do
@@ -50,7 +50,6 @@ class TestPersist < Test::Unit::TestCase
             dumper.add key, key + " - 2"
           end
           dumper.close
-          Thread.exit
         end
         dumper
       end

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -48,15 +48,8 @@ class TestTSVParallelThrough < Test::Unit::TestCase
   def test_traverse_stream
     require 'rbbt/sources/organism'
 
-    head = 100
+    head = 1000
 
-    tsv = Organism.identifiers("Hsa").open
-    res = {}
-    TSV.traverse tsv, :head => head do |k,v|
-       res[k] = v
-    end
-
-    assert_equal head, res.keys.compact.sort.length
     tsv = Organism.identifiers("Hsa").open
     res = {}
     TSV.traverse tsv, :head => head, :into => res do |k,v|
@@ -69,7 +62,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
   def test_traverse_stream_cpus
     require 'rbbt/sources/organism'
 
-    head = 100
+    head = 1000
 
     tsv = Organism.identifiers("Hsa")
     res = {}
@@ -83,7 +76,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
   def test_traverse_stream_keys
     require 'rbbt/sources/organism'
 
-    head = 100
+    head = 1000
 
     tsv = Organism.identifiers("Hsa").open
     res = []
@@ -92,7 +85,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
        res << v
     end
 
-    assert_equal res, Organism.identifiers("Hsa").tsv(:head => 100).keys
+    assert_equal res, Organism.identifiers("Hsa").tsv(:head => head).keys
 
     tsv = Organism.identifiers("Hsa").open
     res = []
@@ -101,7 +94,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
       v
     end
 
-    assert_equal res, Organism.identifiers("Hsa").tsv(:head => 100).keys
+    assert_equal res.sort, Organism.identifiers("Hsa").tsv(:head => head).keys.sort
   end
   
   def test_traverse_array
@@ -165,7 +158,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
   def test_traverse_benchmark
     require 'rbbt/sources/organism'
 
-    head = 8_000
+    head = 2_000
 
     tsv = Organism.identifiers("Hsa").open
     Misc.benchmark do
@@ -182,6 +175,153 @@ class TestTSVParallelThrough < Test::Unit::TestCase
         [k,v]
       end
     end
+  end
+
+  def test_traverse_into_dumper
+    require 'rbbt/sources/organism'
+
+    head = 2_000
+
+    stream = Organism.identifiers("Hsa").open 
+    dumper = TSV::Dumper.new Organism.identifiers("Hsa").tsv_options
+    dumper.init
+    TSV.traverse stream, :head => head, :into => dumper do |k,v|
+      k = k.first
+      [k,v]
+    end
+
+    res = TSV.open(dumper.stream)
+
+    assert_equal head, res.size
+  end
+
+  def test_traverse_into_dumper_threads
+    require 'rbbt/sources/organism'
+
+    head = 2_000
+    threads = 10
+
+    stream = Organism.identifiers("Hsa").open 
+    dumper = TSV::Dumper.new Organism.identifiers("Hsa").tsv_options
+    dumper.init
+
+    TSV.traverse stream, :threads => threads, :head => head, :into => dumper do |k,v|
+      k = k.first
+      [k,v]
+    end
+
+    res = TSV.open(StringIO.new(dumper.stream.read))
+
+    assert_equal head, res.size
+  end
+
+  def test_traverse_into_dumper_cpus
+    require 'rbbt/sources/organism'
+
+    head = 2_000
+    cpus = 10
+
+    stream = Organism.identifiers("Hsa").open 
+    dumper = TSV::Dumper.new Organism.identifiers("Hsa").tsv_options
+    dumper.init
+    TSV.traverse stream, :cpus => cpus, :head => head, :into => dumper do |k,v|
+      k = k.first
+      [k,v]
+    end
+
+    res = TSV.open(dumper.stream)
+
+    assert_equal head, res.size
+  end
+
+  #{{{ TRAVERSE DUMPER
+
+  def test_traverse_dumper
+    require 'rbbt/sources/organism'
 
+    head = 2_000
+
+    tsv = TSV::Parser.new Organism.identifiers("Hsa").open, :head => head
+    dumper = TSV::Dumper.new tsv.options
+
+    TSV.traverse tsv, :head => head, :into => dumper do |k,v|
+      k = k.first
+      [k,v]
+    end
+    
+    res = {}
+    TSV.traverse dumper.stream, :into => res do |k,v|
+      [k, v.length]
+    end
+
+    assert_equal head, res.size
+  end
+
+  def test_traverse_dumper_threads
+    require 'rbbt/sources/organism'
+
+    head = 2_000
+    threads = 3
+
+    tsv = TSV::Parser.new Organism.identifiers("Hsa").open, :head => head
+    dumper = TSV::Dumper.new tsv.options
+
+    TSV.traverse tsv, :head => head, :threads => threads, :into => dumper do |k,v|
+      k = k.first
+      [k,v]
+    end
+    
+    res = {}
+    TSV.traverse dumper.stream, :threads => threads, :into => res do |k,v|
+      [k, v.length]
+    end
+
+    assert_equal head, res.size
+  end
+
+  def test_traverse_dumper_cpus
+    require 'rbbt/sources/organism'
+
+    head = 10_000
+    cpus = nil
+
+    stream = Organism.identifiers("Hsa").open 
+    dumper = TSV::Dumper.new Organism.identifiers("Hsa").tsv_options
+
+    TSV.traverse stream, :head => head, :cpus => cpus, :into => dumper do |k,v|
+      k = k.first
+      [k,v]
+    end
+
+    res = {}
+    TSV.traverse dumper.stream, :cpus => cpus, :into => res do |k,v|
+      [k, v.length]
+    end
+
+    assert_equal head, res.size
+  end
+
+  def test_traverse_dumper_cpus_exception
+    require 'rbbt/sources/organism'
+
+    head = 2_000
+    cpus = 2
+
+    stream = Organism.identifiers("Hsa/jun2011").open 
+    dumper = TSV::Dumper.new Organism.identifiers("Hsa/jun2011").tsv_options
+
+    assert_raise do
+      begin
+      TSV.traverse stream, :head => head,  :cpus => cpus, :into => dumper do |k,v|
+        k = k.first
+        raise "STOP" if rand(100) < 1
+        [k,v]
+      end
+      dumper.stream.read
+      rescue Exception
+        Log.exception $!
+        raise $!
+      end
+    end
   end
 end

data/test/rbbt/util/concurrency/test_processes.rb

@@ -27,7 +27,6 @@ class TestConcurrencyProcess < Test::Unit::TestCase
     end
 
     q.join
-    q.clean
 
     assert_equal times, res.length
     assert_equal [0, 2, 4], res.sort[0..2]

data/test/rbbt/util/test_misc.rb

@@ -115,17 +115,154 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal 4, Misc.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
   end
 
-#  def test_pdf2text_example
-#    assert PDF2Text.pdf2text(datafile_test('example.pdf')).read =~ /An Example Paper/i
-#  end
-#
-#  def test_pdf2text_EPAR
-#    assert PDF2Text.pdf2text("http://www.ema.europa.eu/docs/en_GB/document_library/EPAR_-_Scientific_Discussion/human/000402/WC500033103.pdf").read =~ /Tamiflu/i
-#  end
-#
-#  def test_pdf2text_wrong
-#    assert_raise CMD::CMDError do PDF2Text.pdf2text("http://www.ema.europa.eu/docs/en_GB#").read end
-#  end
+  def test_pipe
+    sout, sin = Misc.pipe
+    assert_equal 1, Misc::OPEN_PIPE_IN.length
+    sin.close
+    assert sout.eof?
+    Misc.purge_pipes
+    assert_equal 0, Misc::OPEN_PIPE_IN.length
+  end
+
+  def test_pipe_fork
+    sout, sin = Misc.pipe
+    pid = Process.fork do
+      Misc.purge_pipes(sin)
+      sleep 2
+      sin.close
+    end
+    sin.close
+    assert sout.eof?
+    Process.kill :INT, pid
+  end
+
+  def test_open_pipe
+    t = 5
+    stream = Misc.open_pipe do |sin|
+      t.times do |i|
+        sleep 0.5
+        sin.puts "LINE #{ i }"
+      end
+    end
+
+    time = Time.now
+    lines = []
+    while line = stream.gets
+      lines << line.strip
+    end
+    time_spent = Time.new - time
+
+    assert time_spent >= t * 0.5
+    assert time_spent <= (t+1) * 0.5
+    assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }"}, lines
+  end
+
+  def test_open_pipe_fork
+    t = 5
+    stream = Misc.open_pipe(true) do |sin|
+      t.times do |i|
+        sleep 0.5
+        sin.puts "LINE #{ i }"
+      end
+    end
+
+    time = Time.now
+    lines = []
+    while line = stream.gets
+      lines << line.strip
+    end
+    time_spent = Time.new - time
+
+    assert time_spent >= t * 0.5
+    assert time_spent <= (t+1) * 0.5
+    assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }"}, lines
+  end
+
+  def test_open_pipe_fork_cascade
+    t = 500
+    sleep_time = 2.0 / t
+    time = Time.now
+
+    stream1 = Misc.open_pipe(true) do |sin|
+      t.times do |i|
+        sleep sleep_time 
+        sin.puts "LINE #{ i }"
+      end
+    end
+
+    stream2 = Misc.open_pipe(true) do |sin|
+      while line = stream1.gets
+        sin.puts line.strip.reverse
+      end
+    end
+
+    stream3 = Misc.open_pipe(true) do |sin|
+      while line = stream2.gets
+        sin.puts line.downcase
+      end
+    end
+
+    lines = []
+    while line = stream3.gets
+      lines << line.strip
+    end
+
+    time_spent = Time.new - time
+
+    assert time_spent >= t * sleep_time
+    assert time_spent <= t * 1.2 * sleep_time
+    assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }".reverse.downcase}, lines
+  end
+
+  def test_tee_stream
+    t = 500
+    sleep_time = 2.0 / t
+    time = Time.now
+
+    stream1 = Misc.open_pipe(true) do |sin|
+      t.times do |i|
+        sleep sleep_time 
+        sin.puts "LINE #{ i }"
+      end
+    end
+
+    stream2, stream3 = Misc.tee_stream stream1
+
+    stream4 = Misc.open_pipe(true) do |sin|
+      while line = stream2.gets
+        sin.puts line.strip.reverse
+      end
+    end
+
+    stream5 = Misc.open_pipe(true) do |sin|
+      while line = stream3.gets
+        sin.puts line.strip.downcase
+      end
+    end
+
+    lines1 = []
+    th1 = Thread.new do
+      while line = stream4.gets
+        lines1 << line.strip
+      end
+    end
+    
+    lines2 = [] 
+    th2 = Thread.new do
+      while line = stream5.gets
+        lines2 << line.strip
+      end
+    end
+    th1.join and th2.join
+
+    time_spent = Time.new - time
+
+    assert time_spent >= t * sleep_time
+    assert time_spent <= t * 1.2 * sleep_time
+    assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }".reverse}, lines1
+    assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }".downcase}, lines2
+  end
+
 
   def test_string2hash
     assert(Misc.string2hash("--user-agent=firefox").include? "--user-agent")
@@ -294,26 +431,6 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal "COSMIC", Misc.camel_case("COSMIC")
   end
 
-  def test_pipe
-    t = 5
-    stream = Misc.open_pipe do |sin|
-      t.times do |i|
-        sleep 0.5
-        sin.puts "LINE #{ i }"
-      end
-    end
-
-    time = Time.now
-    lines = []
-    while line = stream.gets
-      lines << line.strip
-    end
-    time_spent = Time.new - time
-
-    assert time_spent >= t * 0.5
-    assert time_spent <= (t+1) * 0.5
-    assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }"}, lines
-  end
 
   def __test_lock_fd
     require 'rbbt/workflow'