scout-gear 7.1.0 → 7.3.0

data/test/scout/tsv/persist/test_adapter.rb

@@ -0,0 +1,44 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVAdapter < Test::Unit::TestCase
+  def test_get_set
+    tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
+    tsv.type = :list
+    tsv.extend TSVAdapter
+    tsv["a"] = %w(1 2 3)
+
+    assert_equal %w(1 2 3) * "\t", tsv.dup["a"]
+    assert_equal %w(a), tsv.keys
+    assert_equal [%w(1 2 3)], tsv.collect{|k,v| v }
+    assert_equal [%w(1 2 3)], tsv.values
+
+    json = tsv.to_json
+    new = JSON.parse(json)
+    tsv.annotate(new)
+    new.extend TSVAdapter
+
+    tsv = new
+    assert_equal %w(1 2 3) * "\t", tsv.dup["a"]
+    assert_equal %w(a), tsv.keys
+    assert_equal [%w(1 2 3)], tsv.collect{|k,v| v }
+    assert_equal [%w(1 2 3)], tsv.values
+    assert_equal [["a", %w(1 2 3)]], tsv.sort
+
+    tsv["b"] = %w(11 22 33)
+    assert_equal [["a", %w(1 2 3)], ["b", %w(11 22 33)]], tsv.sort
+    assert_equal [["b", %w(11 22 33)], ["a", %w(1 2 3)]], tsv.sort_by{|k,v| -v[0].to_i }
+  end
+
+  def test_serializer
+    tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
+    tsv.type = :list
+    tsv.extend TSVAdapter
+    tsv.serializer = :integer_array
+    tsv["a"] = [1, 2, 3]
+
+    assert_equal [1, 2, 3], tsv["a"]
+  end
+end
+

data/test/scout/tsv/persist/test_fix_width_table.rb

@@ -0,0 +1,134 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestFixWidthTable < Test::Unit::TestCase
+  def load_data(data)
+    tsv = TSV.open(data, type: :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
+    tsv.add_field "Start" do |key, values|
+      values["Range"].first
+    end
+    tsv.add_field "End" do |key, values|
+      values["Range"].last
+    end
+
+    tsv = tsv.slice ["Start", "End"]
+
+    tsv
+  end
+
+  def test_options
+    TmpFile.with_file do |filename|
+      f = FixWidthTable.new filename, 100, true
+      f.close
+
+      f1 = FixWidthTable.new filename, 100, false
+
+      assert_equal true, f1.range
+    end
+  end
+
+  def test_add
+    TmpFile.with_file do |filename|
+      f = FixWidthTable.new filename, 100, true
+      f.add [1,2,0], "test1"
+      f.add [3,4,0], "test2"
+      f.read
+
+      assert_equal 1, f.idx_pos(0)
+      assert_equal 3, f.idx_pos(1)
+      assert_equal 2, f.idx_pos_end(0)
+      assert_equal 4, f.idx_pos_end(1)
+      assert_equal 0, f.idx_overlap(0)
+      assert_equal 0, f.idx_overlap(1)
+      assert_equal "test1", f.idx_value(0)
+      assert_equal "test2", f.idx_value(1)
+
+    end
+
+  end
+
+  def test_point
+    data =<<-EOF
+#: :sep=/\\s+/#:type=:single#:cast=:to_i
+#ID Pos
+a 1
+b 10
+c 20
+d 12
+e 26
+f 11
+g 25
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = TSV.open datafile
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, false
+        f.add_point tsv
+        f.read
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[10].sort
+        assert_equal %w(a b c d f), f[(0..20)].sort
+      end
+    end
+  end
+
+  def test_range
+    data =<<-EOF
+##012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[1].sort
+        assert_equal %w(), f[20].sort
+        assert_equal %w(), f[(20..100)].sort
+        assert_equal %w(a b d), f[3].sort
+        assert_equal %w(a b c d e), f[(3..4)].sort
+        assert_equal %w(a c e), f[7].sort
+      end
+    end
+  end
+
+
+  def test_range_pos
+    data =<<-EOF
+##012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f.overlaps(0).sort
+        assert_equal %w(1:6), f.overlaps(1).sort
+        assert_equal %w(1:6:b), f.overlaps(1, true).sort
+      end
+    end
+  end
+end
+

data/test/scout/tsv/persist/test_tokyocabinet.rb

@@ -0,0 +1,92 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVTokyo < Test::Unit::TestCase
+  def test_tokyo
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      Persist.persist(__method__, :HDB) do
+        TSV.open(filename)
+      end
+    end
+
+    assert_equal %w(a aa aaa), tsv["row1"][0]
+
+    tsv_loaded = assert_nothing_raised do
+      TmpFile.with_file(content) do |filename|
+        Persist.persist(__method__, :HDB) do
+          raise
+        end
+      end
+    end
+
+    assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
+  end
+
+  def test_custom_load
+    tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+
+    size = 100_000
+    (0..size).each do |i|
+      k = "key-#{i}"
+      values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+      values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+      tsv[k] = [values1, values2]
+    end
+
+    tc = Persist.persist(__method__, :HDB) do |file|
+      tsv
+    end
+
+    100.times do
+      i = rand(size).floor
+      assert_equal tc["key-#{i}"], tsv["key-#{i}"]
+    end
+  end
+
+  def test_speed
+    tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+
+    size = 100_000
+    (0..size).each do |i|
+      k = "key-#{i}"
+      values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+      values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+      tsv[k] = [values1, values2]
+    end
+
+    tc = Persist.persist(__method__, :HDB) do |file|
+      data = ScoutCabinet.open(file, true, "HDB")
+      TSV.setup(data, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+      data.extend TSVAdapter
+      Log::ProgressBar.with_bar size do |b|
+        (0..size).each do |i|
+          b.tick
+          k = "key-#{i}"
+          values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+          values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+          data[k] = [values1, values2]
+        end
+      end
+      data
+    end
+
+    100.times do
+      i = rand(size).floor
+      assert_equal tc["key-#{i}"], tsv["key-#{i}"]
+    end
+  end
+end
+

data/test/scout/tsv/test_dumper.rb

@@ -0,0 +1,44 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVDumper < Test::Unit::TestCase
+  def test_dumper
+    dumper = TSV::Dumper.new :key_field => "Key", :fields => %w(Field1 Field2), :type => :double
+    dumper.init
+    dumper.add "a", [["1", "11"], ["2", "22"]]
+    txt=<<-EOF
+#: :type=:double
+#Key\tField1\tField2
+a\t1|11\t2|22
+    EOF
+    dumper.close
+    assert_equal txt, dumper.stream.read
+  end
+
+  def test_to_s
+    tsv = TSV.setup({}, :key_field => "Key", :fields => %w(Field1 Field2), :type => :double)
+    tsv["a"] = [["1", "11"], ["2", "22"]]
+    txt=<<-EOF
+#: :type=:double
+#Key\tField1\tField2
+a\t1|11\t2|22
+    EOF
+    assert_equal txt, tsv.to_s
+  end
+
+  def test_raise
+    dumper = TSV::Dumper.new :key_field => "Key", :fields => %w(Field1 Field2), :type => :double
+    dumper.init
+    t = Thread.new do
+      dumper.add "a", [["1", "11"], ["2", "22"]]
+      dumper.abort ScoutException
+    end
+
+    assert_raise ScoutException do
+      dumper.stream.read
+    end
+  end
+end
+

data/test/scout/tsv/test_index.rb

@@ -0,0 +1,156 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVIndex < Test::Unit::TestCase
+  def load_segment_data(data)
+    tsv = TSV.open(data, type: :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
+
+    tsv = tsv.add_field "Start" do |key, values|
+      values["Range"].first
+    end
+
+    tsv = tsv.add_field "End" do |key, values|
+      values["Range"].last
+    end
+
+    tsv = tsv.slice ["Start", "End"]
+ 
+    tsv
+  end
+
+  def test_index
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3|a
+row2    a    b    id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "ValueB")
+      assert_equal 'b', index["row1"]
+      assert_equal 'b', index["a"]
+      assert_equal 'b', index["aaa"]
+      assert_equal 'B', index["A"]
+    end
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "ValueB", :fields => ["OtherID"])
+      assert_equal 'B', index["a"]
+      assert_nil index["B"]
+    end
+  end
+
+  def test_from_tsv
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3|a
+row2    a    b    id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename)
+      index = TSV.index(tsv, :target => "ValueB", :fields => "OtherID")
+      assert_equal 'B', index["a"]
+      assert_nil index["B"]
+
+      index = tsv.index(:target => "ValueB", :fields => "OtherID")
+      assert_equal 'B', index["a"]
+      assert_nil index["B"]
+    end
+  end
+
+
+  def test_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3|a
+row2    a    b    id3
+    EOF
+    tsv = TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "ValueB", :persist => true)
+      assert_equal 'b', index["row1"]
+      assert_equal 'b', index["a"]
+      assert_equal 'b', index["aaa"]
+      assert_equal 'B', index["A"]
+    end
+  end
+
+  def test_range_index
+    data =<<-EOF
+# 012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_segment_data(datafile)
+      f   = tsv.range_index("Start", "End", :persist => true)
+
+      assert_equal %w(), f[0].sort
+      assert_equal %w(b), f[1].sort
+      assert_equal %w(), f[20].sort
+      assert_equal %w(), f[(20..100)].sort
+      assert_equal %w(a b d), f[3].sort
+      assert_equal %w(a b c d e), f[(3..4)].sort
+    end
+  end
+
+  def test_range_index_persistent
+    data =<<-EOF
+# 012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      load_segment_data(datafile)
+      TmpFile.with_file(load_segment_data(datafile).to_s) do |tsvfile|
+        f = TSV.range_index(tsvfile, "Start", "End", :persist => true)
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[1].sort
+        assert_equal %w(), f[20].sort
+        assert_equal %w(), f[(20..100)].sort
+        assert_equal %w(a b d), f[3].sort
+        assert_equal %w(a b c d e), f[(3..4)].sort
+      end
+    end
+  end
+
+
+  def __test_speed
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3|a
+row2    a    b    id3
+    EOF
+    tsv = TmpFile.with_file(content) do |filename|
+      Misc.benchmark 1000 do
+        TSV.index(filename, :target => "ValueB")
+      end
+      Misc.benchmark 1000 do
+        TSV.index(filename, :target => "ValueB", order: false)
+      end
+    end
+  end
+end
+

data/test/scout/tsv/test_open.rb

@@ -0,0 +1,9 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOpenTraverse < Test::Unit::TestCase
+  def test_true
+    assert true
+  end
+end
+

data/test/scout/tsv/test_parser.rb

@@ -12,6 +12,15 @@ class TestTSVParser < Test::Unit::TestCase
     assert_equal (1..10).collect{|v| v.to_s }, values
   end
 
+  def test_parse_line_key
+    line = (0..10).to_a * "\t"
+    key, values = TSV.parse_line(line, key: 2)
+    
+    assert_equal "2", key
+    assert_equal %w(0 1 3 4 5 6 7 8 9 10), values
+  end
+
+
   def test_parse_double
     line = (0..10).collect{|v| v == 0 ? v : [v,v] * "|" } * "\t"
     key, values = TSV.parse_line(line, type: :double, cast: :to_i)
@@ -20,15 +29,16 @@ class TestTSVParser < Test::Unit::TestCase
     assert_equal (1..10).collect{|v| [v,v] }, values
   end
 
-  def __test_benchmark
+  def ___test_benchmark
     num = 10_000
     txt = num.times.inject(nil) do |acc,i|
       (acc.nil? ? "" : acc << "\n") << (0..10).collect{|v| v == 0 ? i : [v,v] * "|" } * "\t"
     end 
 
     txt = StringIO.new(([txt] * (10))*"\n")
-    Misc.benchmark 1 do
-    #Misc.profile do
+    Misc.benchmark 5 do
+      txt.rewind
+      #Misc.profile do
       data = TSV.parse_stream(txt, fix: true, type: :double, bar: true, merge: :concat)
       assert_equal num, data.size
       assert_equal 20, data['1'][0].length
@@ -84,4 +94,105 @@ k a|A b|B
     tsv = TSV.parse(header)
     assert_equal 'a', tsv['k'][0][0]
   end
+
+  def test_parse_fields
+    content =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+    EOF
+    content = StringIO.new content
+
+    tsv = TSV.parse(content, fields: %w(ValueB))
+    assert_equal [%w(b B)], tsv['k']
+    assert_equal %w(ValueB), tsv.fields
+
+    content.rewind
+
+    tsv = TSV.parse(content, fields: %w(ValueB ValueA))
+    assert_equal [%w(b B), %w(a A)], tsv['k']
+    assert_equal %w(ValueB ValueA), tsv.fields
+
+    content.rewind
+
+    tsv = TSV.parse(content, fields: %w(ValueB Key))
+    assert_equal [%w(b B), %w(k)], tsv['k']
+  end
+
+  def test_parse_key
+    content =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+    EOF
+    content = StringIO.new content
+
+    tsv = TSV.parse(content, key_field: "ValueB")
+    assert_equal %w(b B), tsv.keys
+    assert_equal %w(a A), tsv["B"][1]
+
+    content.rewind
+
+    tsv = TSV.parse(content, key_field: "ValueB", one2one: true, type: :double)
+    assert_equal %w(b B), tsv.keys
+    assert_equal %w(A), tsv["B"][1]
+
+    content.rewind
+
+    tsv = TSV.parse(content, key_field: "ValueB", one2one: true, type: :list)
+    assert_equal %w(b B), tsv.keys
+    assert_equal "a", tsv["b"][1]
+    assert_equal "A", tsv["B"][1]
+    assert_equal "k", tsv["b"][0]
+    assert_equal "k", tsv["B"][0]
+
+    content.rewind
+
+    tsv = TSV.parse(content, key_field: "ValueB", one2one: true, type: :list)
+    assert_equal %w(b B), tsv.keys
+    assert_equal "A", tsv["B"][1]
+  end
+
+  def test_parser_class
+    content =<<-EOF
+Key ValueA ValueB
+k a|A b|B
+    EOF
+    content = StringIO.new content
+
+    parser = TSV::Parser.new content, sep: " ", header_hash: ''
+
+    assert_equal "Key", parser.key_field
+
+    values = []
+    parser.traverse fields: %w(ValueB), type: :double do |k,v|
+      values << [k,v]
+    end
+
+    assert_equal [["k", [%w(b B)]]], values
+  end
+
+
+  def test_parse_persist_serializer
+    content =<<-EOF
+Key ValueA ValueB
+k 1 2
+    EOF
+    content = StringIO.new content
+
+    TmpFile.with_file do |db|
+      data = ScoutCabinet.open db, true, "HDB"
+      TSV.parse content, sep: " ", header_hash: '', data: data, cast: :to_i, type: :list
+      assert_equal [1, 2], data["k"]
+    end
+
+    TmpFile.with_file do |db|
+      content.rewind
+      data = ScoutCabinet.open db, true, "HDB"
+      TSV.parse content, sep: " ", header_hash: '', data: data, cast: :to_i, type: :list, serializer: :float_array
+      assert_equal [1.0, 2.0], data["k"]
+    end
+
+  end
+
 end

data/test/scout/tsv/test_persist.rb

@@ -0,0 +1,43 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVPersist < Test::Unit::TestCase
+  def test_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      TmpFile.with_file(content) do |filename|
+        TSV.open(filename)
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      TmpFile.with_file(content) do |filename|
+        TSV.open(filename)
+      end
+    end
+
+
+    assert_nothing_raised do
+      tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+        raise
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+end
+