scout-gear 10.3.0 → 10.6.1

data/test/scout/persist/tsv/adapter/test_sharder.rb

@@ -0,0 +1,290 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/persist/tsv'
+class TestSharder < Test::Unit::TestCase
+  def test_open_sharder
+    TmpFile.with_file do |dir|
+      shard_function = Proc.new do |key|
+        key[-1]
+      end
+
+      size = 10
+      sharder = Persist.open_sharder(dir, true, :HDB, :path => dir, :shard_function => shard_function, :persist => true, :serializer => :float_array)
+      size.times do |v| 
+        sharder[v.to_s] = [v, v*2]
+      end
+      assert_equal dir, sharder.persistence_path
+      assert_equal size, sharder.size
+
+      assert_equal [2,4], sharder["2"]
+      count = 0
+      sharder.each do |k,v|
+        count += 1
+      end
+      assert_equal count, size
+
+      TSV.setup(sharder, :key_field => "Letter", :fields => ["Value", "Value times 2"], :type => :list)
+
+      assert_equal "Letter", sharder.key_field
+      assert_equal [2,4], sharder["2"]
+      assert_equal 2, sharder["2"]["Value"]
+
+      sharder = Sharder.new dir, false do |key|
+        key[-1]
+      end
+
+      sharder.extend ShardAdapter
+      sharder.load_annotation_hash
+      assert_equal size, sharder.keys.length
+      assert_equal [2,4], sharder["2"]
+
+      sharder = Persist.open_sharder dir do |key|
+        key[-1]
+      end
+
+      assert_equal "Letter", sharder.key_field
+      assert_equal [2,4], sharder["2"]
+      assert_equal 2, sharder["2"]["Value"]
+    end
+  end
+
+  def test_shard_tsv
+    content =<<-EOF
+#Id,ValueA,ValueB
+id1,a1,b1
+id2,a2,b2
+id3,a3,b3
+id11,a11,b11
+    EOF
+
+    TmpFile.with_file(content.gsub(',',"\t")) do |tsv_file|
+      sharder = Persist.tsv(tsv_file, persist_options: { shard_function: proc{|k| k[-1] } }) do |data|
+        TSV.open(tsv_file, data: data, type: :list)
+      end
+      assert_equal 'a1', sharder["id1"]["ValueA"]
+    end
+  end
+
+  def test_shard_tsv_BDB
+    content =<<-EOF
+#Id,ValueA,ValueB
+id1,a1,b1
+id2,a2,b2
+id3,a3,b3
+id11,a11,b11
+    EOF
+
+    TmpFile.with_file(content.gsub(',',"\t")) do |tsv_file|
+      sharder = Persist.tsv(tsv_file, engine: :BDB, persist_options: { shard_function: proc{|k| k[-1] } }) do |data|
+        TSV.open(tsv_file, data: data, type: :list)
+      end
+      assert_equal 'a1', sharder["id1"]["ValueA"]
+      assert_equal %w(id1 id11), sharder.prefix('id1')
+    end
+  end
+
+  def test_shard_fwt
+    TmpFile.with_file do |dir|
+      shard_function = Proc.new do |key|
+        key[0..(key.index(":")-1)]
+      end
+
+      pos_function = Proc.new do |key|
+        key.split(":").last.to_i
+      end
+
+      size = 10
+      sharder = Persist.tsv(dir,
+                            persist_options: {
+                              :engine => 'fwt', 
+                              :path => dir, 
+                              :serializer => :float,
+                              :range => false, :value_size => 64, 
+                              :shard_function => shard_function, 
+                              :pos_function => pos_function
+                            }
+                           ) do |db|
+                             size.times do |v| 
+                               v = v + 1
+                               chr = "chr" << (v % 5).to_s
+                               key = chr + ":" << v.to_s
+                               value = v*2
+                               db[key] = value
+                             end
+                           end
+
+      sharder.read
+
+      assert_equal dir, sharder.persistence_path
+      assert_equal size, sharder.size
+
+      assert_equal 4.0, sharder["chr2:2"]
+
+      count = 0
+      sharder.through do |k,v|
+        count += 1
+      end
+      assert_equal count, size
+
+      sharder = Persist.open_sharder(dir, false, 'fwt', {:range => false, :value_size => 64, :pos_function => pos_function}, &shard_function)
+
+      assert_equal 4.0, sharder["chr2:2"]
+
+      assert_equal size, sharder.size 
+    end
+  end
+
+  def test_shard_pki
+    TmpFile.with_file do |dir|
+      shard_function = Proc.new do |key|
+        key[0..(key.index(":")-1)]
+      end
+
+      pos_function = Proc.new do |key|
+        key.split(":").last.to_i
+      end
+
+      size = 10
+      chrs = (1..10).to_a
+      sharder = Persist.tsv(dir, 
+                            engine: 'pki', 
+                            :persist_options => { 
+                              :pattern =>  %w(f f),
+                              :range => false, 
+                              :value_size => 64, 
+                              :file => dir, 
+                              :shard_function => shard_function,
+                              :pos_function => pos_function
+                            }) do |db|
+        chrs.each do |c|
+          size.times do |v| 
+            v = v 
+            chr = "chr" << c.to_s
+            key = chr + ":" << v.to_s
+            db[key] = [v, v*2]
+          end
+        end
+      end
+      sharder.read
+
+      assert_equal dir, sharder.persistence_path
+
+      db = sharder.database("chr2:2")
+      db.read
+      assert_equal [2.0, 4.0], sharder["chr2:2"]
+
+      assert_equal size*chrs.length, sharder.size
+
+
+      count = 0
+      sharder.through do |k,v|
+        count += 1
+      end
+      assert_equal count, size*chrs.length
+
+      sharder = Persist.open_sharder(
+        dir, false, 'pki', 
+        {:pattern => %w(f f), :file => dir, :range => false, :value_size => 64, :pos_function => pos_function}, &shard_function
+      )
+
+      db = sharder.database("chr2:2")
+      assert_equal [2.0, 4.0], sharder["chr2:2"]
+
+      assert_equal size*chrs.length, sharder.size
+    end
+  end
+
+  def test_shard_pki_skip
+    TmpFile.with_file do |dir|
+      shard_function = Proc.new do |key|
+        key[0..(key.index(":")-1)]
+      end
+
+      pos_function = Proc.new do |key|
+        key.split(":").last.to_i
+      end
+
+      size = 10
+      chrs = (1..10).to_a
+      sharder = Persist.tsv(dir, persist_options: {:pattern => %w(f), :range => false, :value_size => 64, :engine => 'pki', :file => dir, :shard_function => shard_function, :pos_function => pos_function}) do |db|
+        chrs.each do |c|
+          size.times do |v| 
+            v = v + 1
+            chr = "chr" << c.to_s
+            key = chr + ":" << (v*2).to_s
+            db[key] = [v*2]
+          end
+        end
+      end
+      sharder.read
+
+      assert_equal dir, sharder.persistence_path
+
+      assert_equal [2.0], sharder["chr2:2"]
+      assert_equal [4.0], sharder["chr2:4"]
+
+      count = 0
+      sharder.through do |k,v|
+        count += 1 unless v.nil?
+      end
+      assert_equal count, size*chrs.length
+
+      sharder = Persist.open_sharder(dir, false, 'pki', {:range => false, :value_size => 64, :pos_function => pos_function}, &shard_function)
+
+      assert_equal [2.0], sharder["chr2:2"]
+
+    end
+  end
+
+  def test_shard_fwt_persist_tsv
+    TmpFile.with_file do |dir|
+      shard_function = Proc.new do |key|
+        key[0..(key.index(":")-1)]
+      end
+
+      pos_function = Proc.new do |key|
+        key.split(":").last.to_i
+      end
+
+      size = 10
+      sharder = Persist.persist_tsv("ShardTSV_FWT", nil, {}, {
+                              :engine => 'fwt', 
+                              :path => dir, 
+                              :serializer => :float,
+                              :range => false, :value_size => 64, 
+                              :shard_function => shard_function, 
+                              :pos_function => pos_function
+                            }
+                           ) do |db|
+                             size.times do |v| 
+                               v = v + 1
+                               chr = "chr" << (v % 5).to_s
+                               key = chr + ":" << v.to_s
+                               value = v*2
+                               db[key] = value
+                             end
+                           end
+
+      sharder.read
+
+      assert_equal size, sharder.size
+
+      assert_equal 4.0, sharder["chr2:2"]
+
+      count = 0
+      sharder.through do |k,v|
+        count += 1
+      end
+      assert_equal count, size
+
+      sharder = Persist.open_sharder(sharder.persistence_path, false, 'fwt', {:range => false, :value_size => 64, :pos_function => pos_function}, &shard_function)
+
+      assert_equal 4.0, sharder["chr2:2"]
+
+      assert_equal size, sharder.size 
+    end
+  end
+
+end
+

data/test/scout/persist/tsv/adapter/test_tkrzw.rb

@@ -0,0 +1,126 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+
+require 'scout/tsv'
+
+begin
+  require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+  class TestScoutTKRZW < Test::Unit::TestCase
+    def test_open
+      TmpFile.with_file nil do |tmp|
+        db = ScoutTKRZW.open(tmp, true)
+        1000.times do |i|
+          db["foo#{i}"] = "bar#{i}"
+        end
+        assert_include db, 'foo1'
+        assert_equal 1000, db.keys.length
+
+        db.close
+        TmpFile.with_file nil do |tmp_alt|
+          Open.cp tmp, tmp_alt
+          db2 = ScoutTKRZW.open(tmp_alt, false)
+          assert_include db2, 'foo1'
+          assert_equal 1000, db2.keys.length
+        end
+      end
+    end
+
+    def test_tkrzw
+      content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+      EOF
+
+      tsv = TmpFile.with_file(content) do |filename|
+        Persist.persist(__method__, 'tkh') do
+          TSV.open(filename)
+        end
+      end
+
+
+      assert_equal %w(a aa aaa), tsv["row1"][0]
+
+      assert TSVAdapter === tsv
+      assert TSV === tsv
+      assert_include tsv.instance_variable_get(:@annotations), :key_field
+      assert_include tsv.instance_variable_get(:@annotations), :serializer
+
+      tsv.close
+      tsv_loaded = assert_nothing_raised do
+        TmpFile.with_file(content) do |filename|
+          Persist.persist(__method__, 'tkh') do
+            raise
+          end
+        end
+      end
+
+      assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
+    end
+
+    def __test_benchmark1
+      TmpFile.with_file nil do |tmp|
+
+        Misc.benchmark(1000) do
+          db = ScoutTKRZW.open(tmp, true)
+          1000.times do |i|
+            db["foo#{i}"] = "bar#{i}"
+          end
+          10.times do
+            db.keys
+          end
+          10.times do |i|
+            db["foo#{i}"]
+          end
+          Open.rm tmp
+        end
+
+        Misc.benchmark(1000) do
+          db = ScoutCabinet.open(tmp, true)
+          1000.times do |i|
+            db["foo#{i}"] = "bar#{i}"
+          end
+          10.times do
+            db.keys
+          end
+          10.times do |i|
+            db["foo#{i}"]
+          end
+          db.close
+          Open.rm tmp
+        end
+      end
+    end
+
+    def __test_benchmark2
+      TmpFile.with_file nil do |tmp|
+
+        db = ScoutTKRZW.open(tmp, true)
+        10000.times do |i|
+          db["foo#{i}"] = "bar#{i}"
+        end
+
+        Misc.benchmark(1000) do
+          100.times do |i|
+            db["foo#{i}"]
+          end
+        end
+
+        Open.rm tmp
+        db = ScoutCabinet.open(tmp, true)
+        10000.times do |i|
+          db["foo#{i}"] = "bar#{i}"
+        end
+        Misc.benchmark(1000) do
+          100.times do |i|
+            db["foo#{i}"]
+          end
+        end
+      end
+    end
+
+  end
+rescue Exception
+end
+

data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb

@@ -0,0 +1,282 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVTokyo < Test::Unit::TestCase
+  def test_tokyo
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      Persist.persist(__method__, :HDB) do
+        TSV.open(filename)
+      end
+    end
+
+    assert_equal %w(a aa aaa), tsv["row1"][0]
+
+    assert TSVAdapter === tsv
+    assert TSV === tsv
+    assert_include tsv.instance_variable_get(:@annotations), :key_field
+    assert_include tsv.instance_variable_get(:@annotations), :serializer
+
+    tsv_loaded = assert_nothing_raised do
+      TmpFile.with_file(content) do |filename|
+        Persist.persist(__method__, :HDB) do
+          raise
+        end
+      end
+    end
+
+
+    assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
+  end
+
+  def test_custom_load
+    tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+
+    size = 100_000
+    (0..size).each do |i|
+      k = "key-#{i}"
+      values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+      values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+      tsv[k] = [values1, values2]
+    end
+
+    tc = Persist.persist(__method__, :HDB) do |file|
+      tsv
+    end
+
+    100.times do
+      i = rand(size).floor
+      assert_equal tc["key-#{i}"], tsv["key-#{i}"]
+    end
+  end
+
+  def __test_benchmark
+    tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+
+    size = 100_000
+    (0..size).each do |i|
+      k = "key-#{i}"
+      values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+      values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+      tsv[k] = [values1, values2]
+    end
+
+    tc = Persist.persist(__method__, :HDB) do |file|
+      data = ScoutCabinet.open(file, true, "HDB")
+      TSV.setup(data, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+      data.extend TSVAdapter
+      Log::ProgressBar.with_bar size do |b|
+        (0..size).each do |i|
+          b.tick
+          k = "key-#{i}"
+          values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+          values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+          data[k] = [values1, values2]
+        end
+      end
+      data
+    end
+
+    100.times do
+      i = rand(size).floor
+      assert_equal tc["key-#{i}"], tsv["key-#{i}"]
+    end
+  end
+
+  def test_float_array
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   0.2   0.3 0
+row2    0.1  4.5 0
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f, :persist_update => true)
+      assert_equal [0.2, 0.3, 0], tsv["row1"]
+      assert_equal TSVAdapter::FloatArraySerializer, tsv.serializer
+      Open.cp tsv.persistence_path, tmpdir.persistence.foo
+      tsv2 = ScoutCabinet.open(tmpdir.persistence.foo, false)
+      tsv2.extend TSVAdapter
+      assert_equal [0.2, 0.3, 0], tsv2["row1"]
+      assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
+    end
+ 
+  end
+
+  def test_float_double
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   0.2   0.3 0
+row2    0.1  4.5 0
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :double, :cast => :to_f)
+      assert_equal Marshal, tsv.serializer
+      assert_equal [[0.2], [0.3], [0.0]], tsv["row1"]
+      tsv.close
+
+      Persist::CONNECTIONS.clear
+
+      tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :double, :cast => :to_f)
+      assert_equal Marshal, tsv.serializer
+      assert_equal [[0.2], [0.3], [0.0]], tsv["row1"]
+    end
+  end
+
+  def test_importsv_list
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   0.2   0.3 0
+row2    0.1  4.5 0
+    EOF
+
+    TmpFile.with_path(content.gsub(/ +/,"\t")) do |filename|
+      TmpFile.with_file do |persistence_path|
+        parser = TSV::Parser.new filename, type: :list
+        database = ScoutCabinet.open persistence_path, true, :HDB
+        parser.with_stream do |stream|
+          ScoutCabinet.importtsv(database, stream)
+        end
+        database.write_and_read do
+          TSV.setup(database, **parser.options)
+          database.extend TSVAdapter
+        end
+
+        assert_equal '0.2', database["row1"]["ValueA"]
+      end
+    end
+  end
+
+  def test_importsv_double
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   a|aa   b|bb    c|cc
+row2   A|AA   B|BB   C|CC
+    EOF
+
+    TmpFile.with_path(content.gsub(/ +/,"\t")) do |filename|
+      TmpFile.with_file do |persistence_path|
+        parser = TSV::Parser.new filename, type: :double
+        database = ScoutCabinet.open persistence_path, true, :HDB
+        parser.with_stream do |stream|
+          ScoutCabinet.importtsv(database, stream)
+        end
+        database.write_and_read do
+          TSV.setup(database, **parser.options)
+          database.extend TSVAdapter
+        end
+
+        assert_equal %w(A AA), database["row2"]["ValueA"]
+      end
+    end
+  end
+
+  def test_importsv_large
+    content =<<-EOF.gsub(/ +/, "\t")
+#Id    ValueA    ValueB
+    EOF
+
+    10_000.times do |i|
+      content += "row#{i}\ta#{i}\tb#{i}\n"
+    end
+
+    TmpFile.with_path(content) do |filename|
+      TmpFile.with_file do |persistence_path|
+        parser = TSV::Parser.new filename, type: :list
+        database = ScoutCabinet.open persistence_path, true, :HDB
+        parser.with_stream do |stream|
+          ScoutCabinet.importtsv(database, stream)
+        end
+        database.write_and_read do
+          TSV.setup(database, **parser.options)
+          database.extend TSVAdapter
+        end
+
+        assert_equal "a1000", database["row1000"]["ValueA"]
+      end
+    end
+  end
+
+  def test_importsv_double_BDB
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   a|aa   b|bb    c|cc
+row2   A|AA   B|BB   C|CC
+    EOF
+
+    TmpFile.with_path(content.gsub(/ +/,"\t")) do |filename|
+      TmpFile.with_file do |persistence_path|
+        parser = TSV::Parser.new filename, type: :double
+        database = ScoutCabinet.open persistence_path, true, :BDB
+        parser.with_stream do |stream|
+          ScoutCabinet.importtsv(database, stream)
+        end
+        database.write_and_read do
+          TSV.setup(database, **parser.options)
+          database.extend TSVAdapter
+        end
+
+        assert_equal %w(A AA), database["row2"]["ValueA"]
+        assert_equal %w(row1 row2), database.prefix("row")
+      end
+    end
+  end
+
+  def test_importsv_from_file
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   a|aa   b|bb    c|cc
+row2   A|AA   B|BB   C|CC
+    EOF
+
+    TmpFile.with_path(content.gsub(/ +/,"\t")) do |filename|
+      TmpFile.with_file do |persistence_path|
+        parser = TSV::Parser.new filename, type: :double
+        database = ScoutCabinet.open persistence_path, true, :HDB
+        parser.with_stream do |stream|
+          database.load_stream stream
+        end
+        database.write_and_read do
+          TSV.setup(database, **parser.options)
+          database.extend TSVAdapter
+        end
+
+        assert_equal %w(A AA), database["row2"]["ValueA"]
+      end
+    end
+  end
+
+  def test_open_tokyocabinet
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   a|aa   b|bb    c|cc
+row2   A|AA   B|BB   C|CC
+    EOF
+
+    TmpFile.with_path(content.gsub(/ +/,"\t")) do |tsv|
+      TmpFile.with_path do |filename|
+        tk = Persist.open_tokyocabinet(filename, true)
+        TSV.setup(tk, key_field: "Id", fields: %w(ValueA ValueB OtherID), type: :double)
+        tk.save_annotation_hash
+
+        tk = Persist.open_tokyocabinet(filename, true)
+        assert_equal "Id", tk.key_field
+      end
+    end
+  end
+
+end
+

data/test/scout/persist/tsv/test_serialize.rb

@@ -0,0 +1,12 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_serializer_module
+    m = TSVAdapter.serializer_module :marshal
+    v = [1, :a]
+    d = m.dump(v)
+    assert_equal v, m.load(d) 
+  end
+end
+