rbbt-util 5.10.2 → 5.11.1

data/share/rbbt_commands/benchmark/throughput

@@ -17,7 +17,7 @@ EOF
 
 SOPT.usage if options[:help]
 
-last = start = Time.now
+start = Time.now
 count = 0
 max = 0
 avg = 0
@@ -25,11 +25,12 @@ all = []
 scale = 5
 while line = STDIN.gets
   count += 1
+  last ||= Time.now
   if Time.now - last >= 1.0 / scale
     Log.clear_line
-    puts "#{ count*scale } per second. Max #{max*scale}. Average #{avg*scale}"
-    last = Time.now
+    puts "Reading #{ Log.color :blue, count*scale } per second. Max #{ Log.color :blue, max*scale }. Average #{Log.color :blue, avg*scale}"
     max = count > max ? count : max
+    last = Time.now
     all << count
     avg = Misc.mean(all).to_i if all.length > 3
     count = 0
@@ -38,4 +39,4 @@ end
 
 all << count
 
-puts "Total #{Misc.sum(all).to_i} in #{(Time.now - start).to_i} seconds -- #{(Misc.sum(all).to_f / (Time.now - start)).to_i } per second. Max #{max*scale}. Average #{avg*scale}"
+puts "Total #{Log.color :blue, Misc.sum(all).to_i} in #{Log.color :blue, (Time.now - start).to_i} seconds -- #{Log.color :green, (Misc.sum(all).to_f / (Time.now - start)).to_i } per second. Max #{Log.color :blue, max*scale}. Average #{Log.color :green, avg*scale}"

data/share/rbbt_commands/tsv/slice

@@ -23,9 +23,14 @@ SOPT.usage if options[:help]
 
 file = ARGV.shift
 
-file = STDIN if file == '-'
-
-raise ParameterException, "Please specify the tsv file as argument" if file.nil?
+file = case file
+       when "-"
+         STDIN
+       when String
+         Open.open(file)
+       else
+         raise ParameterException, "Please specify the tsv file as argument" if file.nil?
+       end
 
 fields = options[:fields]
 raise ParameterException, "Please specify the fields to slice" if fields.nil?
@@ -40,7 +45,11 @@ when options[:tokyocabinet_bd]
   tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
   puts tsv.summary
 else
-  tsv = TSV.open(file, options)
+  stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :keys => fields, :unnamed => true) do |*p|
+    p * "\t"
+  end
+  puts stream.read
+  exit 0
 end
 
 puts tsv.slice(fields.split(","))

data/share/rbbt_commands/workflow/task

@@ -333,7 +333,7 @@ begin
   end
 
   if options.delete(:printpath)
-    job.join
+    job.join.load
     puts job.path
     exit 0
   end
@@ -403,12 +403,18 @@ when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
   out.puts res.load
 when Step
   if IO === res.result
-    io = res.result
-    Thread.pass while IO.select([io]).nil?
-    while block = io.read(2048) do
-      out.write block
-    end unless io.closed? 
-    io.join if io.respond_to? :join
+    begin
+      io = res.result
+      Thread.pass while IO.select([io]).nil?
+      while block = io.read(2048) do
+        out.write block
+      end unless io.closed? 
+      io.join if io.respond_to? :join
+    rescue Exception
+      io.abort if io.respond_to? :abort
+    ensure
+      io.join if io.respond_to? :join
+    end
   else
     res.join
     out.puts Open.read(res.path) if File.exists? res.path

data/test/rbbt/test_persist.rb

@@ -12,7 +12,7 @@ end
 
 class TestPersist < Test::Unit::TestCase
 
-  def _test_array_persist
+  def test_array_persist
     TmpFile.with_file do |tmp|
       10.times do
         assert_equal ["1", "2"],(Persist.persist("Test", :array, :file => tmp) do
@@ -40,7 +40,7 @@ class TestPersist < Test::Unit::TestCase
 
   def test_tsv_dumper
     TmpFile.with_file do |tmpdir|
-      stream = Persist.persist("Dumper", :tsv, :dir => tmpdir) do
+      tsv = Persist.persist("Dumper", :tsv, :dir => tmpdir) do
         dumper = TSV::Dumper.new :key_field => "Field 1", :fields => ["Field 2"], :type => :single
 
         dumper.init
@@ -54,10 +54,27 @@ class TestPersist < Test::Unit::TestCase
         dumper
       end
 
-      while line = stream.gets do
-        puts line
+      assert_equal 10, tsv.size
+    end
+  end
+
+  def test_tsv_dumper_stream
+    TmpFile.with_file do |tmpdir|
+      stream = Persist.persist("Dumper", :tsv, :dir => tmpdir, :no_load => :stream) do
+        dumper = TSV::Dumper.new :key_field => "Field 1", :fields => ["Field 2"], :type => :single
+
+        Thread.new do
+          10.times do |i|
+            key = i.to_s
+            dumper.add key, key + " - 2"
+          end
+          dumper.close
+        end
+
+        dumper
       end
 
+      assert_equal 10, stream.read.split("\n").length
     end
   end
 end

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -324,4 +324,13 @@ class TestTSVParallelThrough < Test::Unit::TestCase
       end
     end
   end
+
+  def test_traverse_into_stream
+    size = 100
+    array = (1..size).to_a.collect{|n| n.to_s}
+    stream = TSV.traverse array, :into => :stream do |e|
+      e
+    end
+    assert_equal size, stream.read.split("\n").length
+  end
 end

data/test/rbbt/util/concurrency/processes/test_socket.rb

@@ -28,7 +28,9 @@ class TestConcurrency < Test::Unit::TestCase
     end
 
     socket.swrite.close
-    assert ClosedStream  === socket.pop
+    assert_raise ClosedStream do 
+      socket.pop
+    end
 
     socket.clean
   end

data/test/rbbt/util/concurrency/test_processes.rb

@@ -44,6 +44,35 @@ class TestConcurrencyProcess < Test::Unit::TestCase
       assert_equal times, Dir.glob(File.join(dir, '*')).length
     end
   end
+
+  def test_process_abort
+    assert_raise Aborted do
+      q = RbbtProcessQueue.new 10
+
+      res = []
+
+      q.callback do |v|
+        res << v
+      end
+
+      q.init do |i|
+        sleep 1 while true
+      end
+
+      times = 500
+      t = TSV.setup({"a" => 1}, :type => :single)
+
+      times.times do |i|
+        q.process i
+      end
+
+      sleep 2
+      q.clean
+
+      q.join
+
+    end
+  end
 end
 
 

data/test/rbbt/util/test_misc.rb

@@ -117,11 +117,11 @@ eum fugiat quo voluptas nulla pariatur?"
 
   def test_pipe
     sout, sin = Misc.pipe
-    assert_equal 1, Misc::OPEN_PIPE_IN.length
+    assert_equal 1, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
     sin.close
     assert sout.eof?
     Misc.purge_pipes
-    assert_equal 0, Misc::OPEN_PIPE_IN.length
+    assert_equal 0, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
   end
 
   def test_pipe_fork
@@ -325,10 +325,10 @@ eum fugiat quo voluptas nulla pariatur?"
       pids = []
       4.times do |i|
         pids << Process.fork do 
-          pid = Process.pid().to_s
-          status = Misc.lock(tmpfile, pid) do |f, val|
-            Open.write(f, val)
-            sleep rand * 2
+          status = Misc.lock(tmpfile) do 
+            pid = Process.pid.to_s
+            Open.write(tmpfile, pid)
+            sleep rand * 1
             if pid == Open.read(tmpfile)
               0
             else

data/test/rbbt/workflow/test_step.rb

@@ -122,7 +122,7 @@ class TestStep < Test::Unit::TestCase
         step = Step.new tmp, task
         job = step.fork
         assert !job.done?
-        assert_raise RuntimeError do step.fork end
+        assert_raise RuntimeError do step.clean.fork end
         sleep 1
         while not job.abort do sleep 1 end
         Open.write(lock, "open")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.10.2
+  version: 5.11.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-03 00:00:00.000000000 Z
+date: 2014-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake