rbbt-util 5.13.37 → 5.14.0

data/share/rbbt_commands/workflow/task

@@ -269,8 +269,6 @@ end
 
 if recursive_clean 
   job.recursive_clean 
-  sleep 1
-  job = workflow.job(task.name, name, job_options)
 end
 
 require 'pp'

data/test/rbbt/persist/tsv/test_sharder.rb

@@ -75,5 +75,49 @@ class TestSharder < Test::Unit::TestCase
       assert_equal size, sharder.size 
     end
   end
+
+  def test_shard_pki
+    TmpFile.with_file do |dir|
+      shard_function = Proc.new do |key|
+        key[0..(key.index(":")-1)]
+      end
+
+      pos_function = Proc.new do |key|
+        key.split(":").last.to_i
+      end
+
+      size = 10
+      chrs = (1..10).to_a
+      sharder = Persist.persist_tsv(nil, "ShardTest", {}, :pattern => %w(f), :update => true, :range => false, :value_size => 64, :engine => 'pki', :file => dir, :shard_function => shard_function, :pos_function => pos_function, :persist => true, :serializer => :clean) do |db|
+        chrs.each do |c|
+          size.times do |v| 
+            v = v 
+            chr = "chr" << c.to_s
+            key = chr + ":" << v.to_s
+            iii [key, v]
+            db << [key, [v*2]]
+          end
+        end
+      end
+      sharder.read
+
+      assert_equal dir, sharder.persistence_path
+      assert_equal size*chrs.length, sharder.size
+
+      assert_equal [4.0], sharder["chr2:2"]
+
+      count = 0
+      sharder.through do |k,v|
+        count += 1
+      end
+      assert_equal count, size*chrs.length
+
+      sharder = Persist.open_sharder(dir, false, :float, 'fwt', {:range => false, :value_size => 64, :pos_function => pos_function}, &shard_function)
+
+      assert_equal [4.0], sharder["chr2:2"]
+
+      assert_equal chrs.length*size, sharder.size 
+    end
+  end
 end
 

data/test/rbbt/test_fix_width_table.rb

@@ -44,6 +44,7 @@ class TestFixWidthTable < Test::Unit::TestCase
       assert_equal 0, f.overlap(1)
       assert_equal "test1", f.value(0)
       assert_equal "test2", f.value(1)
+
     end
  
   end

data/test/rbbt/test_packed_index.rb

@@ -13,12 +13,15 @@ class TestPackedIndex < Test::Unit::TestCase
       pi << nil
       pi.close
       pi = PackedIndex.new tmpfile, false
+      Misc.benchmark(1000) do
       100.times do |i|
         assert_equal i, pi[i][0] 
         assert_equal i+2, pi[i][1] 
       end
+      end
       assert_equal nil, pi[100]
       assert_equal nil, pi[101]
+
     end
   end
 end

data/test/rbbt/tsv/test_stream.rb

@@ -90,6 +90,38 @@ row2 cc
     assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
   end
 
+  def test_paste_stream_missing_2
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb 
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "", "", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["", "", "", "", "", "ccc"], tsv["row3"]
+  end
+
   def test_paste_stream_missing
     text1=<<-EOF
 #: :sep=" "
@@ -101,7 +133,6 @@ row1 A B C
     text2=<<-EOF
 #: :sep=" "
 #Row Labela Labelb 
-row1 a b
 row2 aa bb
     EOF
 
@@ -118,8 +149,30 @@ row2 cc
     tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
     assert_equal "Row", tsv.key_field
     assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
-    assert_equal ["A", "B", "C", "a", "b", ""], tsv["row1"]
+    assert_equal ["A", "B", "C", "", "", ""], tsv["row1"]
     assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
     assert_equal ["", "", "", "", "", "ccc"], tsv["row3"]
   end
+
+  def test_paste_stream_missing_3
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labelc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.paste_streams([s1,s2], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labelc), tsv.fields
+    assert_equal ["A", "B", "C", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", ""], tsv["row2"]
+  end
 end

data/test/rbbt/util/misc/test_pipes.rb

@@ -5,7 +5,7 @@ require 'rbbt/util/misc'
 
 class TestMiscPipes < Test::Unit::TestCase
 
-  def _test_collapse_stream
+  def test_collapse_stream
     text=<<-EOF
 row1 A B C
 row1 a b c
@@ -19,7 +19,7 @@ row2 aa bb cc
     assert_equal ["BB", "bb"], tsv["row2"][1]
   end
 
-  def _test_paste_stream
+  def test_paste_stream
     text1=<<-EOF
 row1 A B C
 row2 AA BB CC
@@ -46,9 +46,11 @@ row3 ccc
     assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
   end
 
-  def _test_sort_stream
+  def test_sort_stream
     text =<<-EOF
-#: :sep=" "
+##
+##
+##
 #Row LabelA LabelB LabelC
 row2 AA BB CC
 row3 AAA BBB CCC
@@ -56,8 +58,8 @@ row1 A B C
     EOF
     s = StringIO.new text
     sorted = Misc.sort_stream(s)
-    assert_equal %w(#: #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
-    assert_equal %w(#: #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
   end
 
   def test_dup_stream

data/test/rbbt/workflow/test_step.rb

@@ -192,7 +192,7 @@ class TestStep < Test::Unit::TestCase
         task  = Task.setup do 
           5.times do
             puts "Process: #{Process.pid}"
-            sleep rand * 2
+            sleep rand 
           end
         end
 
@@ -203,11 +203,12 @@ class TestStep < Test::Unit::TestCase
             jobs << step.fork(semaphore)
           end
         end
-        jobs.each do |job|
-          while not job.done?
-            sleep 1
-          end
-        end
+        Step.wait_for_jobs(jobs)
+        #jobs.each do |job|
+        #  while not job.done?
+        #    sleep 1
+        #  end
+        #end
       ensure
         RbbtSemaphore.delete_semaphore(semaphore)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.37
+  version: 5.14.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-12 00:00:00.000000000 Z
+date: 2014-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -140,6 +140,7 @@ files:
 - lib/rbbt/persist/tsv/kyotocabinet.rb
 - lib/rbbt/persist/tsv/leveldb.rb
 - lib/rbbt/persist/tsv/lmdb.rb
+- lib/rbbt/persist/tsv/packed_index.rb
 - lib/rbbt/persist/tsv/sharder.rb
 - lib/rbbt/persist/tsv/tokyocabinet.rb
 - lib/rbbt/resource.rb