scout-gear 10.3.0 → 10.6.1

data/test/scout/persist/engine/test_packed_index.rb

@@ -0,0 +1,99 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_packed_index
+
+    TmpFile.with_file do |tmpfile|
+      pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
+      100.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
+      pi = PackedIndex.new(tmpfile, false)
+      100.times do |i|
+        assert_equal i, pi[i][0] 
+        assert_equal i+2, pi[i][1] 
+      end
+      assert_equal nil, pi[100]
+      assert_equal nil, pi[101]
+    end
+  end
+
+  def __test_benchmark_bgzip
+    size = 1000000
+    density = 0.1
+
+    access = []
+    (size * density).to_i.times do
+      access << rand(size-1) + 1
+    end
+    access.sort!
+    access.uniq!
+
+    TmpFile.with_file do |tmpfile|
+      pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
+      size.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
+
+      pi = PackedIndex.new(tmpfile, false)
+      Misc.benchmark do
+        access.each do |point|
+          assert_equal point+2, pi[point][1]
+        end
+      end
+
+      `bgzip #{tmpfile} `
+      `mv #{tmpfile}.gz #{tmpfile}.bgz`
+
+      pi = PackedIndex.new(tmpfile + '.bgz', false)
+      pi[0]
+      Misc.benchmark do
+        access.each do |point|
+          assert_equal point+2, pi[point][1]
+        end
+      end
+    end
+  end
+
+  def __test_benchmark
+
+    TmpFile.with_file do |tmpfile|
+      pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
+      100.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
+      pi = PackedIndex.new(tmpfile, false)
+      Misc.benchmark(1000) do
+        100.times do |i|
+          assert_equal i, pi[i][0] 
+          assert_equal i+2, pi[i][1] 
+        end
+      end
+
+      TmpFile.with_file do |tmpfile|
+        tk = Persist.open_tokyocabinet tmpfile, true, :json
+        100.times do |i|
+          tk[i] = [i, i+2, (i.to_s * 10).to_i, rand, rand, rand, rand, rand]
+        end
+        Misc.benchmark(1000) do
+          100.times do |i|
+            assert_equal i, tk[i][0] 
+            assert_equal i+2, tk[i][1] 
+          end
+        end
+      end
+    end
+  end
+
+end
+

data/test/scout/persist/engine/test_sharder.rb

@@ -0,0 +1,31 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/persist'
+
+class TestSharder < Test::Unit::TestCase
+  def test_shard_open
+
+    TmpFile.with_file do |tmpfile|
+      sharder = Sharder.new tmpfile, true, :HDB do |key|
+        key.to_s[-1]
+      end
+
+      sharder["key-a"] = "a"
+      sharder["key-b"] = "b"
+      assert_equal "a", sharder["key-a"]
+      assert_equal "b", sharder["key-b"]
+
+      sharder = Sharder.new tmpfile, true, :HDB
+      sharder.shard_function = proc do |key|
+        key.split("-").last
+      end
+
+      assert_equal "a", sharder["key-a"]
+      assert_equal "b", sharder["key-b"]
+
+      assert_equal 2, sharder.persistence_path.glob("*").select{|f| f.include?("shard-") }.length
+    end
+  end
+end
+

data/test/scout/persist/engine/test_tokyocabinet.rb

@@ -0,0 +1,17 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestScoutCabinet < Test::Unit::TestCase
+  def test_open
+    TmpFile.with_file do |tmpfile|
+      db = ScoutCabinet.open(tmpfile)
+      db["a"] = 1
+      assert_equal "1", db["a"]
+      db.close
+
+      db = ScoutCabinet.open(tmpfile, false)
+      assert_equal "1", db["a"]
+    end
+  end
+end
+

data/test/scout/persist/test_tsv.rb

@@ -0,0 +1,146 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVPersist < Test::Unit::TestCase
+  def test_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      TmpFile.with_file(content) do |filename|
+        TSV.open(filename)
+      end
+    end
+
+    assert NamedArray === tsv["row1"]
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      TmpFile.with_file(content) do |filename|
+        TSV.open(filename)
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
+    assert_nothing_raised do
+      tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+        raise
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+
+  def test_persist_with_data
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = nil
+    TmpFile.with_file do |tk|
+      data = Persist.open_tokyocabinet(tk, true, "HDB")
+      assert Open.exists?(tk)
+      tsv = Persist.persist("TEST Persist TSV", :HDB, :persist_data => data) do |data|
+        t = TmpFile.with_file(content) do |filename|
+          TSV.open(filename, persist_data: data)
+        end
+        t
+        nil
+      end
+      refute Open.exists?(tk)
+      assert Open.exists?(data.persistence_path)
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
+    assert_nothing_raised do
+      tsv = Persist.persist("TEST Persist TSV", :HDB) do 
+        raise
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+
+  def test_tsv
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = nil
+    TmpFile.with_file(content) do |filename|
+      tsv = Persist.tsv("Some TSV") do |data|
+        TSV.open(filename, persist_data: data)
+      end
+      assert_equal ['b'], tsv["row1"][1]
+      assert NamedArray === tsv["row1"]
+      assert_equal ['b'], tsv["row1"]["ValueB"]
+      assert_include tsv.keys, 'row1'
+      assert_include tsv.keys, 'row2'
+      assert_nothing_raised do
+        tsv = Persist.tsv("Some TSV") do |data|
+          raise
+        end
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+
+  def test_persist_tsv
+    content =<<-'EOF'
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = nil
+    TmpFile.with_file(content) do |filename|
+      tsv = Persist.persist_tsv("Some TSV", sep: /\s+/, type: :double) do |data|
+        TSV.open(filename, sep: /\s+/, type: :double, persist_data: data)
+      end
+      assert_equal ['b'], tsv["row1"][1]
+      assert NamedArray === tsv["row1"]
+      assert_equal ['b'], tsv["row1"]["ValueB"]
+      assert_include tsv.keys, 'row1'
+      assert_include tsv.keys, 'row2'
+      assert_nothing_raised do
+        tsv = Persist.persist_tsv("Some TSV", sep: /\s+/, type: :double) do |data|
+          raise
+        end
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+end
+

data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb}

@@ -1,7 +1,6 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-require 'scout/tsv'
 class TestTSVAdapter < Test::Unit::TestCase
   def test_get_set
     tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
@@ -9,7 +8,6 @@ class TestTSVAdapter < Test::Unit::TestCase
     tsv.extend TSVAdapter
     tsv["a"] = %w(1 2 3)
 
-    assert_equal %w(1 2 3) * "\t", tsv.dup["a"]
     assert_equal %w(a), tsv.keys
     assert_equal [%w(1 2 3)], tsv.collect{|k,v| v }
     assert_equal [%w(1 2 3)], tsv.values
@@ -20,7 +18,6 @@ class TestTSVAdapter < Test::Unit::TestCase
     new.extend TSVAdapter
 
     tsv = new
-    assert_equal %w(1 2 3) * "\t", tsv.dup["a"]
     assert_equal %w(a), tsv.keys
     assert_equal [%w(1 2 3)], tsv.collect{|k,v| v }
     assert_equal [%w(1 2 3)], tsv.values
@@ -35,10 +32,12 @@ class TestTSVAdapter < Test::Unit::TestCase
     tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
     tsv.type = :list
     tsv.extend TSVAdapter
-    tsv.serializer = :integer_array
+    tsv.serializer = :marshal
     tsv["a"] = [1, 2, 3]
 
+
     assert_equal [1, 2, 3], tsv["a"]
+    assert_equal [1, 2, 3], Marshal.load(tsv.orig_get("a"))
   end
 end
 

data/test/scout/persist/tsv/adapter/test_fix_width_table.rb

@@ -0,0 +1,46 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestFixWidthTable < Test::Unit::TestCase
+  def load_data(data)
+    tsv = TSV.open(data, type: :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
+    tsv.add_field "Start" do |key, values|
+      values["Range"].first
+    end
+    tsv.add_field "End" do |key, values|
+      values["Range"].last
+    end
+
+    tsv = tsv.slice ["Start", "End"]
+
+    tsv
+  end
+
+  def test_open_fwt
+    data =<<-EOF
+##012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = Persist.open_fwt filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f.overlaps(0).sort
+        assert_equal %w(1:6), f.overlaps(1).sort
+        assert_equal %w(1:6:b), f.overlaps(1, true).sort
+      end
+    end
+  end
+end
+

data/test/scout/persist/tsv/adapter/test_packed_index.rb

@@ -0,0 +1,37 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_open
+    TmpFile.with_file do |tmpfile|
+      pi = Persist.open_pki tmpfile, true, %w(i i 23s f f f f f)
+      100.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
+
+      TSV.setup(pi, :key_field => "Number", :fields => %w(i i2 s f1 f2 f3 f4 f5), :type => :list)
+
+      pi = PackedIndex.new(tmpfile, false)
+      100.times do |i|
+        assert_equal i, pi[i][0] 
+        assert_equal i+2, pi[i][1] 
+      end
+      assert_equal nil, pi[100]
+      assert_equal nil, pi[101]
+
+      pi = Persist.open_pki tmpfile, false, %w(i i 23s f f f f f)
+      100.times do |i|
+        assert_equal i, pi[i][0] 
+        assert_equal i+2, pi[i][1] 
+      end
+      assert_equal nil, pi[100]
+      assert_equal nil, pi[101]
+
+      assert_equal "Number", pi.key_field
+    end
+  end
+end
+