rbbt-util 5.8.10 → 5.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 985a049f7bd611901dc77ac6fbb9f8529f9ed01c
-  data.tar.gz: 653082e1984c57a177161f0e1f23e1a2c1f1506c
+  metadata.gz: 8809e7ff7afbc160340f326450273a14ca3ce286
+  data.tar.gz: 3de26d6f418fd60b21459b9cef3af0ed11844310
 SHA512:
-  metadata.gz: d1ecb5c1fb2a9880b52003a0c20e137fc627ab9ed685dab911ce2f4c84e32bce7fbfad4c9abbb1c245872c325e2d388373ca4abc9e4167ed2887a9e8ed3d14a2
-  data.tar.gz: 5e40f5ac11b3bbb015eb30742770821f6d434442b95aeb62c5afe8eccec0aec6d02601e317e2737d8b1eea0f5c66cc27f0b22df1ec31ee092a3215edb185a0aa
+  metadata.gz: 8f3d7b38e869e5b88aa3005a6cdb1ffb96ca9eb8c8ef96625892ed3ba8a038422b9c7172806bda50fd1e1bdf4e014d75607ad0c8c4fe33a806e0498d3e4e6b88
+  data.tar.gz: c56376cae5c4fc796fedef603179fd4e1129581b7c495e6101705fe939f5facd1c5a200e500d592df10c26ec91466c118b79ca38efc2ad2b14aa9b414cf61be3

data/lib/rbbt/annotations/util.rb

@@ -209,8 +209,8 @@ module Annotated
     end
   end
 
-  def marshal_dump(depth)
-   clean_annotations
+  def marshal_dump
+    Annotated.purge(self).to_sym.to_s
   end
 end
 

data/lib/rbbt/tsv.rb

@@ -160,7 +160,7 @@ module TSV
     data
   end
 
-  def self.traverse(file, options, &block)
+  def self.traverse_old(file, options, &block)
     return file.through(options[:key_field], options[:fields], &block) if TSV === file
 
     TSV::Parser.traverse(TSV.get_stream(file), options, &block)

data/lib/rbbt/tsv/parallel.rb

@@ -1,104 +1,7 @@
 require 'rbbt/util/concurrency'
 
-module TSV
-
-  def pthrough(num_threads = 10, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
-    q = RbbtThreadQueue.new num_threads
-
-    q.init(true, &block)
-
-    begin
-      res = through(new_key_field, new_fields, uniq, zipped) do |*p|
-        q.process p
-      end
-      q.join
-    ensure
-      q.clean
-    end
-
-  end
-
-  def ppthrough_callback(&block)
-    @ppthrough_callback = block
-  end
-
-  def ppthrough(num_procs = 7, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
-
-    q = RbbtProcessQueue.new num_procs
-
-    q.callback &@ppthrough_callback
-    @ppthrough_callback = nil
-
-    q.init do |k,v|
-      block.call k,v
-    end
-
-    begin
-      res = through(new_key_field, new_fields, uniq, zipped) do |*p|
-        q.process q
-      end
-      q.join
-    ensure
-      q.clean
-    end
-
-    res
-  end
+require 'rbbt/tsv/parallel/through'
+require 'rbbt/tsv/parallel/traverse'
 
-  def ppthrough(num_procs = 7, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
-
-    q = RbbtProcessQueue.new num_procs
-
-    q.callback &@ppthrough_callback
-    @ppthrough_callback = nil
-
-    _pat_size = 20
-    _pat = "A" << _pat_size.to_s
-
-    num_fields = fields.length
-    pattern = case type
-              when :single, :flat
-                _pat * 2
-              when :list, :double
-                _pat * (num_fields + 1)
-              end
-
-    q.init do |str|
-      _parts = str.unpack(pattern)
-
-      case type
-      when :single
-        k, v = _parts
-      when :list
-        k, *v = _parts
-      when :flat
-        k, v = _parts
-        v = v.split "|"
-      when :double
-        k, *v = _parts
-        v = v.collect{|l| l.split "|" }
-      end
-
-      block.call k,v
-    end
-
-    begin
-      res = through(new_key_field, new_fields, uniq, zipped) do |k,v|
-        case type
-        when :flat
-          v = v * "|" 
-        when :double
-          v = v.collect{|l| l * "|" } if type == :double
-        end
-
-        str = [k,v].flatten.pack(pattern)
-        q.process str
-      end
-      q.join
-    ensure
-      q.clean
-    end
-
-    res
-  end
+module TSV
 end

data/lib/rbbt/tsv/parallel/through.rb

@@ -0,0 +1,102 @@
+module TSV
+
+  def pthrough(num_threads = 10, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
+    q = RbbtThreadQueue.new num_threads
+
+    q.init(true, &block)
+
+    begin
+      res = through(new_key_field, new_fields, uniq, zipped) do |*p|
+        q.process p
+      end
+      q.join
+    ensure
+      q.clean
+    end
+
+  end
+
+  def ppthrough_callback(&block)
+    @ppthrough_callback = block
+  end
+
+  def ppthrough(num_procs = 7, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
+
+    q = RbbtProcessQueue.new num_procs
+
+    q.callback &@ppthrough_callback
+    @ppthrough_callback = nil
+
+    q.init do |k,v|
+      block.call k,v
+    end
+
+    begin
+      res = through(new_key_field, new_fields, uniq, zipped) do |*p|
+        q.process q
+      end
+      q.join
+    ensure
+      q.clean
+    end
+
+    res
+  end
+
+  def ppthrough(num_procs = 7, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
+
+    q = RbbtProcessQueue.new num_procs
+
+    q.callback &@ppthrough_callback
+    @ppthrough_callback = nil
+
+    _pat_size = 20
+    _pat = "A" << _pat_size.to_s
+
+    num_fields = fields.length
+    pattern = case type
+              when :single, :flat
+                _pat * 2
+              when :list, :double
+                _pat * (num_fields + 1)
+              end
+
+    q.init do |str|
+      _parts = str.unpack(pattern)
+
+      case type
+      when :single
+        k, v = _parts
+      when :list
+        k, *v = _parts
+      when :flat
+        k, v = _parts
+        v = v.split "|"
+      when :double
+        k, *v = _parts
+        v = v.collect{|l| l.split "|" }
+      end
+
+      block.call k,v
+    end
+
+    begin
+      res = through(new_key_field, new_fields, uniq, zipped) do |k,v|
+        case type
+        when :flat
+          v = v * "|" 
+        when :double
+          v = v.collect{|l| l * "|" } if type == :double
+        end
+
+        str = [k,v].flatten.pack(pattern)
+        q.process str
+      end
+      q.join
+    ensure
+      q.clean
+    end
+
+    res
+  end
+end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -0,0 +1,138 @@
+module TSV
+
+  def self.traverse_tsv(tsv, options = {}, &block)
+    callback = Misc.process_options options, :callback
+
+    if callback
+      tsv.through options[:key_field], options[:fields] do |k,v|
+        callback.call yield(k,v)
+      end
+    else
+      tsv.through options[:key_field], options[:fields] do |k,v|
+        yield k,v 
+      end
+    end
+  end
+
+  def self.traverse_array(array, options = {}, &block)
+    callback = Misc.process_options options, :callback
+
+    if callback
+      array.each do |e|
+        res = yield(e)
+        callback.call res
+      end
+    else
+      array.each do |e|
+        yield e
+      end
+    end
+  end
+
+  def self.traverse_obj(obj, options = {}, &block)
+    if options[:type] == :keys
+      options[:fields] = []
+      options[:type] = :single
+    end
+
+    case obj
+    when TSV
+      traverse_tsv(obj, options, &block)
+    when IO
+      callback = Misc.process_options options, :callback
+      if callback
+        TSV::Parser.traverse(obj, options) do |k,v|
+          res = yield k, v
+          callback.call res
+        end
+      else
+        TSV::Parser.traverse(obj, options, &block)
+      end
+    when Array
+      traverse_array(obj, options, &block)
+    else
+      raise "Unknown object for traversal: #{Misc.fingerprint obj }"
+    end
+  end
+
+  def self.traverse_threads(num, obj, options, &block)
+    callback = Misc.process_options options, :callback
+
+    q = RbbtThreadQueue.new num
+
+
+    if callback
+      block = Proc.new do |k,v,mutex|
+        v, mutex = nil, v if mutex.nil?
+        res = yield k, v
+        mutex.synchronize do
+          callback.call res
+        end
+      end
+    end
+
+    q.init true, &block
+
+    traverse_obj(obj, options) do |*p|
+      q.process p
+    end
+
+    q.join
+    q.clean
+    nil
+  end
+
+  def self.traverse_cpus(num, obj, options, &block)
+    callback = Misc.process_options options, :callback
+
+    q = RbbtProcessQueue.new num
+
+    q.callback &callback
+    q.init &block
+
+    traverse_obj(obj, options) do |*p|
+      q.process p
+    end
+
+    q.join
+    q.clean
+    nil
+  end
+
+  def self.store_into(obj, value)
+    case obj
+    when Hash
+      if Hash === value
+        obj.merge! value
+      else
+        k,v = value
+        obj[k] = v
+      end
+    else
+      obj << value
+    end 
+  end
+
+  def self.traverse(obj, options = {}, &block)
+    threads = Misc.process_options options, :threads
+    cpus = Misc.process_options options, :cpus
+    into = Misc.process_options options, :into
+
+    if into
+      callback = Proc.new do |*v|
+        store_into into, *v
+      end
+      options[:callback] = callback
+    end
+
+    if threads.nil? and cpus.nil?
+      traverse_obj obj, options, &block
+    else
+      if threads
+        traverse_threads threads, obj, options, &block 
+      else
+        traverse_cpus cpus, obj, options, &block
+      end
+    end
+  end
+end

data/lib/rbbt/util/concurrency/processes.rb

@@ -50,6 +50,7 @@ class RbbtProcessQueue
         while @processes.any? do
           pid = Process.wait -1, Process::WNOHANG
           if pid
+            next unless @processes.collect{|p| p.pid }.include? pid
             @processes.delete_if{|p| p.pid == pid}
             raise "Process #{pid} failed" unless $?.success?
           else

data/lib/rbbt/util/concurrency/threads.rb

@@ -19,7 +19,8 @@ class RbbtThreadQueue
           begin
             loop do
               p = queue.pop
-              p << mutex 
+              p = Array === p ? p << mutex : [p,mutex]
+              p = [p, mutex].flatten(1) 
               block.call *p
             end
           rescue Exception
@@ -66,7 +67,6 @@ class RbbtThreadQueue
     queue << e
   end
 
-
   def self.each(list, num = 3, &block)
     q = RbbtThreadQueue.new num
     q.init(&block)

data/lib/rbbt/util/misc.rb

@@ -101,6 +101,7 @@ module Misc
     end
     str
   end
+
   def self.parse_cmd_params(str)
     return str if Array === str
     str.scan(/

data/lib/rbbt/workflow/accessor.rb

@@ -125,6 +125,8 @@ class Step
   end
 
   def log(status, message = nil, &block)
+    self.status = status
+    self.message Log.uncolor(message)
     Step.log(status, message, path, &block)
   end
 

data/test/rbbt/annotations/test_util.rb

@@ -6,9 +6,8 @@ class TestClass < Test::Unit::TestCase
     a = "STRING"
     a.extend Annotated
 
+    assert !(Annotated === Marshal.load(Marshal.dump(a)))
     assert_equal a, Marshal.load(Marshal.dump(a))
-    assert_equal Marshal.dump("STRING"), Marshal.dump(a)
-
   end
 end
 

data/test/rbbt/persist/tsv/test_cdb.rb

@@ -12,12 +12,14 @@ end
 
 class TestPersistTSVCDB < Test::Unit::TestCase
 
-  def test_organism
-    require 'rbbt/sources/organism'
-    TmpFile.with_file nil do |tmp_file|
-      file = CMD.cmd("head -n 1000000", :in => Organism.identifiers("Hsa").open, :pipe => true)
-      tsv = Organism.identifiers("Hsa").tsv(:key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "CDB", :persist_dir => tmp_file)
-      assert_equal "ENSG00000141510", tsv["TP53"]
+  if Persist.respond_to? :open_cbd
+    def test_organism
+      require 'rbbt/sources/organism'
+      TmpFile.with_file nil do |tmp_file|
+        file = CMD.cmd("head -n 1000000", :in => Organism.identifiers("Hsa").open, :pipe => true)
+        tsv = Organism.identifiers("Hsa").tsv(:key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "CDB", :persist_dir => tmp_file)
+        assert_equal "ENSG00000141510", tsv["TP53"]
+      end
     end
   end
 end

data/test/rbbt/persist/tsv/test_kyotocabinet.rb

@@ -12,20 +12,22 @@ module TestAnnotation
 end
 
 class TestPersistTSVKC < Test::Unit::TestCase
-  def _test_organism_kch
-    require 'rbbt/sources/organism'
-    TmpFile.with_file do |tmp_file|
-      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kch", :persist_dir => tmp_file
-      assert_equal "ENSG00000141510", tsv["TP53"]
+  if Persist.respond_to? :open_kyotocabinet
+    def _test_organism_kch
+      require 'rbbt/sources/organism'
+      TmpFile.with_file do |tmp_file|
+        tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kch", :persist_dir => tmp_file
+        assert_equal "ENSG00000141510", tsv["TP53"]
+      end
     end
-  end
 
 
-  def _test_organism_kct
-    require 'rbbt/sources/organism'
-    TmpFile.with_file do |tmp_file|
-      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kct", :persist_dir => tmp_file
-      assert_equal "ENSG00000141510", tsv["TP53"]
+    def _test_organism_kct
+      require 'rbbt/sources/organism'
+      TmpFile.with_file do |tmp_file|
+        tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kct", :persist_dir => tmp_file
+        assert_equal "ENSG00000141510", tsv["TP53"]
+      end
     end
   end
 end

data/test/rbbt/persist/tsv/test_leveldb.rb

@@ -12,11 +12,13 @@ end
 
 class TestPersistTSVLevelDB < Test::Unit::TestCase
 
-  def test_organism
-    require 'rbbt/sources/organism'
-    TmpFile.with_file do |tmp_file|
-      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "LevelDB", :persist_dir => tmp_file
-      assert_equal "ENSG00000141510", tsv["TP53"]
+  if Persist.respond_to? :open_leveldb
+    def test_organism
+      require 'rbbt/sources/organism'
+      TmpFile.with_file do |tmp_file|
+        tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "LevelDB", :persist_dir => tmp_file
+        assert_equal "ENSG00000141510", tsv["TP53"]
+      end
     end
   end
 end

data/test/rbbt/persist/tsv/test_lmdb.rb

@@ -14,9 +14,11 @@ class TestPersistTSVLMDB < Test::Unit::TestCase
 
   def test_organism
     require 'rbbt/sources/organism'
-    TmpFile.with_file do |tmp_file|
-      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "LMDB", :persist_dir => tmp_file
-      assert_equal "ENSG00000141510", tsv["TP53"]
+    if Persist.respond_to? :open_lmdb
+      TmpFile.with_file do |tmp_file|
+        tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "LMDB", :persist_dir => tmp_file
+        assert_equal "ENSG00000141510", tsv["TP53"]
+      end
     end
   end
 end

data/test/rbbt/test_annotations.rb

@@ -42,19 +42,6 @@ class TestAnnotations < Test::Unit::TestCase
     assert_equal annotation_str, ary[0].annotation_str
   end
 
-  def test_double_array
-    ary = ["string"]
-    annotation_str = "Annotation String"
-    ary.extend AnnotatedArray
-    ary_ary = [ary]
-    ary_ary.extend AnnotatedArray
-    AnnotatedString.setup(ary, annotation_str)
-    AnnotatedString.setup(ary_ary, annotation_str)
-    assert_equal [AnnotatedString], ary.annotation_types
-    assert_equal annotation_str, ary.annotation_str
-    assert_equal annotation_str, ary[0].annotation_str
-  end
-
 
   def test_info
     ary = ["string"]
@@ -145,7 +132,20 @@ class TestAnnotations < Test::Unit::TestCase
     assert_equal str + annotation_str, str.add_annot
   end
 
-  def test_double_array
+  def test_double_array_1
+    ary = ["string"]
+    annotation_str = "Annotation String"
+    ary.extend AnnotatedArray
+    ary_ary = [ary]
+    ary_ary.extend AnnotatedArray
+    AnnotatedString.setup(ary, annotation_str)
+    AnnotatedString.setup(ary_ary, annotation_str)
+    assert_equal [AnnotatedString], ary.annotation_types
+    assert_equal annotation_str, ary.annotation_str
+    assert_equal annotation_str, ary[0].annotation_str
+  end
+
+  def test_double_array_2
     a = ["a"]
     b = AnnotatedString.setup([AnnotatedString.setup(["a"])])
     AnnotatedString.setup(a)

data/test/rbbt/test_workflow.rb

@@ -106,7 +106,7 @@ class TestWorkflow < Test::Unit::TestCase
     assert TestWF.jobs(:repeat2).include?(job1.name)
     assert TestWF.jobs(:repeat2).include?(job2.name)
     assert TestWF.jobs(:repeat2).include?(job3.name)
-    assert TestWF.load_name(:repeat2, TestWF.jobs(:repeat2).first).done?
+    assert TestWF.load_name(:repeat2, job3.name).done?
     assert_equal "TEST\nTEST\nTEST\nTEST\nTEST\nTEST", TestWF.load_name(:repeat2, TestWF.jobs(:repeat2).first).load
   end
 

data/test/rbbt/tsv/parallel/test_through.rb

@@ -0,0 +1,42 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/tsv'
+require 'rbbt/tsv/parallel'
+
+class TestTSVParallelThrough < Test::Unit::TestCase
+
+  def test_pthrough
+    require 'rbbt/sources/organism'
+    tsv = Organism.identifiers("Hsa").tsv :unnamed => true, :persist => false, :fields => ["Associated Gene Name"]
+
+    h = {}
+    tsv.monitor = true
+    tsv.unnamed = true
+    tsv.pthrough do |k,v|
+      h[k] = v.first
+    end
+
+    assert_equal tsv.size, h.size
+    assert_equal tsv.keys.sort, h.keys.sort
+  end
+
+
+  def test_ppthrough
+    require 'rbbt/sources/organism'
+    tsv = Organism.identifiers("Hsa").tsv :unnamed => true, :persist => false, :fields => ["Associated Gene Name"]
+
+    h = {}
+
+    tsv.ppthrough_callback do |k,v|
+      h[k] = v
+    end
+
+    tsv.unnamed = true
+    tsv.monitor = true
+    tsv.ppthrough(3) do |k,v|
+      [k,v.first]
+    end
+
+    assert_equal tsv.size, h.size
+    assert_equal tsv.keys.sort, h.keys.sort
+  end
+end

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -0,0 +1,151 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/tsv'
+require 'rbbt/tsv/parallel'
+
+class TestTSVParallelThrough < Test::Unit::TestCase
+
+  def _test_traverse_tsv
+    require 'rbbt/sources/organism'
+
+    head = 100
+
+    tsv = Organism.identifiers("Hsa").tsv :head => head
+    res = {}
+    TSV.traverse tsv do |k,v|
+       res[k] = v
+    end
+    assert_equal head, res.keys.compact.sort.length
+
+    tsv = Organism.identifiers("Hsa").tsv :head => head
+    TSV.traverse tsv, :into => res do |k,v|
+      [k,v]
+    end
+    assert_equal head, res.keys.compact.sort.length
+  end
+
+  def _test_traverse_stream
+    require 'rbbt/sources/organism'
+
+    head = 100
+
+    tsv = Organism.identifiers("Hsa").open
+    res = {}
+    TSV.traverse tsv, :head => head do |k,v|
+       res[k] = v
+    end
+
+    assert_equal head, res.keys.compact.sort.length
+    tsv = Organism.identifiers("Hsa").open
+    res = {}
+    TSV.traverse tsv, :head => head, :into => res do |k,v|
+      [k,v]
+    end
+
+    assert_equal head, res.keys.compact.sort.length
+  end
+
+  def _test_traverse_stream_keys
+    require 'rbbt/sources/organism'
+
+    head = 100
+
+    tsv = Organism.identifiers("Hsa").open
+    res = []
+
+    TSV.traverse tsv, :head => head, :type => :keys do |v|
+       res << v
+    end
+
+    assert_equal res, Organism.identifiers("Hsa").tsv(:head => 100).keys
+
+    tsv = Organism.identifiers("Hsa").open
+    res = []
+
+    TSV.traverse tsv, :head => head, :type => :keys, :into => res do |v|
+      v
+    end
+
+    assert_equal res, Organism.identifiers("Hsa").tsv(:head => 100).keys
+  end
+  
+  def _test_traverse_array
+    require 'rbbt/sources/organism'
+
+    array = []
+    100.times do array << rand(10).to_i end
+
+    res = []
+    TSV.traverse array do |v|
+       res << v
+    end
+
+    assert_equal array, res
+
+
+    res = []
+    TSV.traverse array, :into => res do |v|
+      v
+    end
+
+    assert_equal array, res
+  end
+
+  def _test_traverse_array_threads
+    require 'rbbt/sources/organism'
+
+    array = []
+    100.times do array << rand(10).to_i end
+
+    res = []
+    TSV.traverse array, :threads => 5 do |v|
+       res << v
+    end
+
+    assert_equal array.sort, res.sort
+
+    res = []
+    TSV.traverse array, :threads => 5, :into => res do |v|
+      v
+    end
+
+    assert_equal array.sort, res.sort
+  end
+
+  def _test_traverse_array_cpus
+    require 'rbbt/sources/organism'
+
+    array = []
+    100.times do array << rand(10).to_i end
+
+    res = []
+
+    TSV.traverse array, :cpus => 5, :into => res do |v|
+      v
+    end
+
+    assert_equal array.sort, res.sort
+  end
+
+  def test_traverse_benchmark
+    require 'rbbt/sources/organism'
+
+    head = 80_000
+
+    tsv = Organism.identifiers("Hsa").open
+    Misc.benchmark do
+      res = {}
+      TSV.traverse tsv, :head => head do |k,v|
+        res[k] = v
+      end
+    end
+
+    tsv = Organism.identifiers("Hsa").open
+    Misc.benchmark do
+      res = {}
+      TSV.traverse tsv, :head => head, :cpus => 5, :into => res do |k,v|
+        [k,v]
+      end
+    end
+
+  end
+end

data/test/rbbt/tsv/test_accessor.rb

@@ -3,7 +3,7 @@ require 'rbbt/tsv'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-class TestTSV < Test::Unit::TestCase
+class TestTSVAccessor < Test::Unit::TestCase
 
   def test_to_hash
     content =<<-EOF

data/test/rbbt/tsv/test_attach.rb

@@ -476,7 +476,8 @@ A    Id3
     end
 
     res = tsv1.attach tsv2, :fields => ["OtherID"]
-    assert_equal "Id3", res["row2"]["OtherID"]
+    assert res["row2"].include? "Id3"
+    assert ! res["row2"].include?("b")
   end
 end
 

data/test/rbbt/tsv/test_index.rb

@@ -190,8 +190,8 @@ row2    A    B    30   35
 
   def test_range_index2
     data =<<-EOF
+# 012345678901234567890
 #ID:Range
-#:012345678901234567890
 a:   ______
 b: ______
 c:    _______
@@ -215,8 +215,8 @@ g:         ____
 
   def test_range_index_persistent
     data =<<-EOF
+# 012345678901234567890
 #ID:Range
-#:012345678901234567890
 a:   ______
 b: ______
 c:    _______
@@ -259,8 +259,8 @@ g:         ____
 
   def test_range_index_persistent_with_filter
     data =<<-EOF
+# 012345678901234567890
 #ID:Range
-#:012345678901234567890
 a:   ______
 b: ______
 c:    _______

data/test/rbbt/tsv/test_parallel.rb

@@ -4,39 +4,4 @@ require 'rbbt/tsv/parallel'
 
 class TestTSVParallel < Test::Unit::TestCase
 
-  def _test_pthrough
-    require 'rbbt/sources/organism'
-    tsv = Organism.identifiers("Hsa").tsv :unnamed => true, :persist => false, :fields => ["Associated Gene Name"]
-
-    h = {}
-    tsv.monitor = true
-    tsv.unnamed = true
-    tsv.pthrough do |k,v|
-      h[k] = v.first
-    end
-
-    assert_equal tsv.size, h.size
-    assert_equal tsv.keys.sort, h.keys.sort
-  end
-
-
-  def test_ppthrough
-    require 'rbbt/sources/organism'
-    tsv = Organism.identifiers("Hsa").tsv :unnamed => true, :persist => false, :fields => ["Associated Gene Name"]
-
-    h = {}
-
-    tsv.ppthrough_callback do |k,v|
-      h[k] = v
-    end
-
-    tsv.unnamed = true
-    tsv.monitor = true
-    tsv.ppthrough(3) do |k,v|
-      [k,v.first]
-    end
-
-    assert_equal tsv.size, h.size
-    assert_equal tsv.keys.sort, h.keys.sort
-  end
 end

data/test/rbbt/util/concurrency/test_processes.rb

@@ -7,7 +7,7 @@ require 'rbbt/util/concurrency/processes'
 class TestConcurrencyProcess < Test::Unit::TestCase
 
   def test_process
-    q = RbbtProcessQueue.new 1
+    q = RbbtProcessQueue.new 10
 
     res = []
 
@@ -22,18 +22,15 @@ class TestConcurrencyProcess < Test::Unit::TestCase
     times = 500
     t = TSV.setup({"a" => 1}, :type => :single)
 
-    Misc.benchmark do
     times.times do |i|
       q.process i
     end
 
     q.join
     q.clean
-    end
 
     assert_equal times, res.length
     assert_equal [0, 2, 4], res.sort[0..2]
-    
   end
 
   def test_each

data/test/rbbt/util/concurrency/test_threads.rb

@@ -11,12 +11,10 @@ class TestConcurrencyThreads < Test::Unit::TestCase
       res << v
     end
 
-    times = 5_000_000
-    Misc.benchmark do
+    times = 1_000_000
     times.times do |i|
       q.process [i*2]
     end
-    end
 
     q.join
     q.clean
@@ -27,7 +25,7 @@ class TestConcurrencyThreads < Test::Unit::TestCase
   end
 
   def test_each
-    times = 50000
+    times = 10000
     elems = (0..times-1).to_a
 
     TmpFile.with_file do |dir|

data/test/rbbt/util/test_misc.rb

@@ -145,30 +145,8 @@ eum fugiat quo voluptas nulla pariatur?"
 
   def test_path_relative_to
     assert_equal "test/foo", Misc.path_relative_to('/test', '/test/test/foo')
-
-    Misc.profile do
-    Misc.benchmark(45_000) do
-      Misc.path_relative_to('/test', '/test/test/foo')
-    end
-    end
   end
 
-#  def test_chunk
-#    test =<<-EOF
-#This is an example file. Entries are separated by Entry
-#-- Entry
-#1
-#2
-#3
-#-- Entry
-#4
-#5
-#6
-#    EOF
-#
-#    assert_equal "1\n2\n3", Misc.chunk(test, /^-- Entry/).first.strip
-#  end
-
   def test_hash2string
     hash = {}
     assert_equal hash, Misc.string2hash(Misc.hash2string(hash))

data/test/rbbt/util/test_simpleDSL.rb

@@ -2,7 +2,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
 require 'rbbt/util/simpleDSL'
 require 'test/unit'
 
-class TestClass 
+class TestSimpleDSLClass 
   include SimpleDSL
 
   def action(name, *args, &block)
@@ -18,7 +18,7 @@ end
 
 class TestDSL < Test::Unit::TestCase
   def setup 
-    @parser = TestClass.new
+    @parser = TestSimpleDSLClass.new
     @parser.load_config(:action) do
       action1 "Hello"
       action2 "Good bye"

data/test/rbbt/workflow/test_step.rb

@@ -210,7 +210,5 @@ class TestStep < Test::Unit::TestCase
         RbbtSemaphore.delete_semaphore(semaphore)
       end
     end
-
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.8.10
+  version: 5.9.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-18 00:00:00.000000000 Z
+date: 2014-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -153,6 +153,8 @@ files:
 - lib/rbbt/tsv/index.rb
 - lib/rbbt/tsv/manipulate.rb
 - lib/rbbt/tsv/parallel.rb
+- lib/rbbt/tsv/parallel/through.rb
+- lib/rbbt/tsv/parallel/traverse.rb
 - lib/rbbt/tsv/parser.rb
 - lib/rbbt/tsv/serializers.rb
 - lib/rbbt/tsv/util.rb
@@ -260,6 +262,8 @@ files:
 - test/rbbt/test_resource.rb
 - test/rbbt/test_tsv.rb
 - test/rbbt/test_workflow.rb
+- test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_change_id.rb
@@ -370,5 +374,7 @@ test_files:
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/test_fix_width_table.rb
 - test/test_helper.rb