scout-gear 10.4.0 → 10.6.1

data/test/scout/entity/test_property.rb

@@ -0,0 +1,345 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/entity'
+require 'scout/tsv'
+
+class TestEntityProperty < Test::Unit::TestCase
+  class TestA
+    attr_accessor :foo, :bar
+    def initialize(foo, bar)
+      @foo = foo
+      @bar = bar
+    end
+  end
+
+  $count = 0
+
+  module ReversableString
+    extend Entity
+
+    self.annotation :foo, :bar
+
+    property :times do |times|
+      [self] * times
+    end
+
+    property :reverse_both => :both do
+      if Array === self
+        self.collect{|s| s.reverse }
+      else
+        self.reverse
+      end
+    end
+
+    property :reverse_text_ary => :array do
+      $count += 1
+      self.collect{|s| s.reverse}
+    end
+
+
+    property :reverse_text_ary_hash => :array do
+      $count += 1
+      res = {}
+      self.each{|s| res[s] = s.reverse }
+      res
+    end
+
+    property :reverse_text_single => :single do
+      $count += 1
+      self.reverse
+    end
+
+    property :reverse_text_ary_p => :array2single do
+      $count += 1
+      self.collect{|s| s.reverse}
+    end
+
+    property :reverse_text_single_p => :single do
+      $count += 1
+      self.reverse
+    end
+
+    property :reverse_text_ary_p_array => :array do
+      $count += 1
+      self.collect{|s| s.reverse }
+    end
+
+    property :random => :single do
+      rand
+    end
+
+    property :annotation_list => :single do
+      self.chars.to_a.collect{|c| 
+        ReversableString.setup(c)
+      }
+    end
+
+    property :annotation_list_empty => :single do
+      []
+    end
+
+    $processed_multiple = []
+    property :multiple_annotation_list => :multiple do 
+      $processed_multiple.concat self
+      res = {}
+      self.collect do |e|
+        e.chars.to_a.collect{|c| 
+          ReversableString.setup(c)
+        }
+      end
+    end
+
+    #persist :multiple_annotation_list, :annotation, :dir => Rbbt.tmp.test.annots
+  end
+
+  setup do
+    ReversableString.persist :reverse_text_ary_p, :marshal
+
+    ReversableString.persist :reverse_text_ary_p_array, :array, :dir => TmpFile.tmp_file
+
+    ReversableString.persist :annotation_list, :annotation, :annotation_repo => TmpFile.tmp_file
+
+    ReversableString.persist :annotation_list_empty, :annotation, :dir => TmpFile.tmp_file
+
+    ReversableString.persist :multiple_annotation_list, :annotation, :dir => TmpFile.tmp_file
+  end
+
+
+  def test_property_ary_make_list
+    $count = 0
+    a = "String1"
+    ReversableString.setup(a)
+
+    assert_equal "1gnirtS", a.reverse_text_ary
+  end
+
+  def test_property_both
+    a = "String1"
+    ReversableString.setup(a)
+
+    $count = 0
+
+    assert_equal "1gnirtS", a.reverse_both
+
+    a = ["String1"]
+    ReversableString.setup(a)
+
+    assert_equal "1gnirtS", a.reverse_both.last
+  end
+
+  def test_property_single_simple
+    a = "String1"
+    ReversableString.setup(a)
+
+    $count = 0
+
+    assert_equal "1gnirtS", a.reverse_text_single
+  end
+
+  def test_property_ary
+    a = ["String1", "String2"]
+    ReversableString.setup(a)
+
+    $count = 0
+
+    assert_equal "2gnirtS", a.reverse_text_ary.last
+    assert_equal 1, $count
+    a._ary_property_cache.clear
+    assert ReversableString === a[1]
+    assert Entity::Object === a[1]
+    assert_equal "2gnirtS", a[1].reverse_text_ary
+    assert_equal 2, $count
+    a._ary_property_cache.clear
+
+    $count = 0
+    a.each do |string|
+      string.reverse_text_ary
+      assert_equal 1, $count
+    end
+  end
+
+  def test_property_ary_hash
+    a = ["String1", "String2"]
+    ReversableString.setup(a)
+
+    $count = 0
+
+    assert_equal "2gnirtS", a.reverse_text_ary_hash["String2"]
+    assert_equal 1, $count
+    a._ary_property_cache.clear
+    assert_equal "2gnirtS", a[1].reverse_text_ary_hash
+    assert_equal 2, $count
+    a._ary_property_cache.clear
+
+    $count = 0
+    a.each do |string|
+      string.reverse_text_ary
+      assert_equal 1, $count
+    end
+  end
+
+  def test_property_single
+    a = ["String1", "String2"]
+    ReversableString.setup a
+
+    $count = 0
+
+    assert_equal "2gnirtS", a.reverse_text_single.last
+    assert_equal 2, $count
+    assert_equal "2gnirtS", a[1].reverse_text_single
+    assert_equal 3, $count
+  end
+
+  def test_property_ary_p
+    a = ["String1", "String2"]
+    ReversableString.setup a
+
+    a.reverse_text_ary_p
+
+    $count = 0
+
+    assert_equal "2gnirtS", a.reverse_text_ary_p.last
+    assert_equal "2gnirtS", a.collect{|e| e.reverse_text_ary_p }[1]
+    assert_equal 0, $count
+  end
+
+  def test_property_single_p
+    a = ["String1", "String2"]
+    ReversableString.setup a
+
+    $count = 0
+
+    assert_equal "2gnirtS", a.reverse_text_single_p.last
+
+    assert_equal 2, $count
+
+    a = ["String1", "String2"]
+    ReversableString.setup a
+
+    $count = 0
+
+    assert_equal "2gnirtS", a.reverse_text_single_p.last
+    assert_equal 2, $count
+    assert_equal "2gnirtS", a[1].reverse_text_single_p
+    assert_equal 3, $count
+  end
+
+  def test_property_ary_p_array
+    a = ["String1", "String2"]
+    ReversableString.setup a
+
+    assert_equal "2gnirtS", a.reverse_text_ary_p_array.last
+
+    $count = 0
+
+    assert_equal "2gnirtS", a.reverse_text_ary_p_array.last
+    assert_equal 0, $count
+    assert_equal "2gnirtS", a.reverse_text_ary_p_array.last
+    assert_equal 0, $count
+  end
+
+  def test_unpersist
+    a = ["String1", "String2"]
+    ReversableString.setup a
+
+    # Before persist
+    assert(! ReversableString.persisted?(:random))
+
+    r1 = a.random
+    r2 = a.random
+    assert_not_equal r1, r2
+
+    # After persist
+    ReversableString.persist :random
+    assert ReversableString.persisted?(:random)
+
+    r1 = a.random
+    r2 = a.random
+    assert_equal r1, r2
+
+    # After unpersist
+    ReversableString.unpersist :random
+    refute ReversableString.persisted?(:random)
+
+    r1 = a.random
+    r2 = a.random
+    assert_not_equal r1, r2
+
+  end
+
+  def test_persist_annotations
+    string = 'aaabbbccc'
+    ReversableString.setup(string)
+    assert_equal string.length, string.annotation_list.length
+    assert_equal string.length, string.annotation_list.length
+  end
+
+  def test_persist_annotations_empty
+    string = 'aaabbbccc'
+    ReversableString.setup(string)
+    assert_equal [], string.annotation_list_empty
+    assert_equal [], string.annotation_list_empty
+  end
+
+  def test_persist_multiple_annotations
+    string1 = 'aaabbbccc'
+    string2 = 'AAABBBCCC'
+    string3 = 'AAABBBCCC_3'
+    string4 = 'AAABBBCCC_4'
+
+
+    $processed_multiple = []
+
+    array = ReversableString.setup([string1, string2])
+    assert_equal [string1, string2].collect{|s| s.chars}, array.multiple_annotation_list
+    assert_equal string1.length, array[0].multiple_annotation_list.length
+    assert_equal [string1, string2], $processed_multiple
+
+    array = ReversableString.setup([string2, string3])
+    assert_equal [string2, string3].collect{|s| s.chars}, array.multiple_annotation_list
+    assert_equal string3, array.multiple_annotation_list.last * ""
+    assert_equal [string1, string2, string3], $processed_multiple
+
+    $processed_multiple = []
+    array = ReversableString.setup([string2, string3])
+    assert_equal [string2, string3].collect{|s| s.chars}, array.multiple_annotation_list
+    assert_equal string2.length, array[0].multiple_annotation_list.length
+    assert_equal [], $processed_multiple
+
+    $processed_multiple = []
+    array = ReversableString.setup([string2, string3, string4])
+    assert_equal string2.length, array.multiple_annotation_list[0].length
+    assert_equal [string4], $processed_multiple
+
+    string1 = 'aaabbbccc'
+    string2 = 'AAABBBCCC'
+    string3 = 'AAABBBCCC_3'
+    string4 = 'AAABBBCCC_4'
+
+    $processed_multiple = []
+    array = ReversableString.setup([string2, string3, string4])
+    assert_equal string2.length, array[0].multiple_annotation_list.length
+    assert_equal $processed_multiple, []
+
+  end
+
+  def test_clean_annotations
+
+    string = "test_string"
+    ReversableString.setup string
+    assert string.respond_to?(:reverse_text_single)
+    assert ! string.purge.respond_to?(:reverse_text_single)
+
+  end
+
+  def test_all_properties
+    assert ReversableString.setup("TEST").all_properties.include?(:reverse_text_ary)
+    assert_equal ReversableString.setup("TEST").all_properties, ReversableString.properties
+  end
+
+  def test_times
+    assert_equal ["TEST", "TEST"], ReversableString.setup("TEST").times(2)
+  end
+end
+

data/test/scout/{tsv/persist → persist/engine}/test_fix_width_table.rb

@@ -1,7 +1,6 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-require 'scout/tsv'
 class TestFixWidthTable < Test::Unit::TestCase
   def load_data(data)
     tsv = TSV.open(data, type: :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})

data/test/scout/persist/engine/test_packed_index.rb

@@ -0,0 +1,99 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_packed_index
+
+    TmpFile.with_file do |tmpfile|
+      pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
+      100.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
+      pi = PackedIndex.new(tmpfile, false)
+      100.times do |i|
+        assert_equal i, pi[i][0] 
+        assert_equal i+2, pi[i][1] 
+      end
+      assert_equal nil, pi[100]
+      assert_equal nil, pi[101]
+    end
+  end
+
+  def __test_benchmark_bgzip
+    size = 1000000
+    density = 0.1
+
+    access = []
+    (size * density).to_i.times do
+      access << rand(size-1) + 1
+    end
+    access.sort!
+    access.uniq!
+
+    TmpFile.with_file do |tmpfile|
+      pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
+      size.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
+
+      pi = PackedIndex.new(tmpfile, false)
+      Misc.benchmark do
+        access.each do |point|
+          assert_equal point+2, pi[point][1]
+        end
+      end
+
+      `bgzip #{tmpfile} `
+      `mv #{tmpfile}.gz #{tmpfile}.bgz`
+
+      pi = PackedIndex.new(tmpfile + '.bgz', false)
+      pi[0]
+      Misc.benchmark do
+        access.each do |point|
+          assert_equal point+2, pi[point][1]
+        end
+      end
+    end
+  end
+
+  def __test_benchmark
+
+    TmpFile.with_file do |tmpfile|
+      pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
+      100.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
+      pi = PackedIndex.new(tmpfile, false)
+      Misc.benchmark(1000) do
+        100.times do |i|
+          assert_equal i, pi[i][0] 
+          assert_equal i+2, pi[i][1] 
+        end
+      end
+
+      TmpFile.with_file do |tmpfile|
+        tk = Persist.open_tokyocabinet tmpfile, true, :json
+        100.times do |i|
+          tk[i] = [i, i+2, (i.to_s * 10).to_i, rand, rand, rand, rand, rand]
+        end
+        Misc.benchmark(1000) do
+          100.times do |i|
+            assert_equal i, tk[i][0] 
+            assert_equal i+2, tk[i][1] 
+          end
+        end
+      end
+    end
+  end
+
+end
+

data/test/scout/persist/engine/test_sharder.rb

@@ -0,0 +1,31 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/persist'
+
+class TestSharder < Test::Unit::TestCase
+  def test_shard_open
+
+    TmpFile.with_file do |tmpfile|
+      sharder = Sharder.new tmpfile, true, :HDB do |key|
+        key.to_s[-1]
+      end
+
+      sharder["key-a"] = "a"
+      sharder["key-b"] = "b"
+      assert_equal "a", sharder["key-a"]
+      assert_equal "b", sharder["key-b"]
+
+      sharder = Sharder.new tmpfile, true, :HDB
+      sharder.shard_function = proc do |key|
+        key.split("-").last
+      end
+
+      assert_equal "a", sharder["key-a"]
+      assert_equal "b", sharder["key-b"]
+
+      assert_equal 2, sharder.persistence_path.glob("*").select{|f| f.include?("shard-") }.length
+    end
+  end
+end
+

data/test/scout/persist/engine/test_tokyocabinet.rb

@@ -0,0 +1,17 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestScoutCabinet < Test::Unit::TestCase
+  def test_open
+    TmpFile.with_file do |tmpfile|
+      db = ScoutCabinet.open(tmpfile)
+      db["a"] = 1
+      assert_equal "1", db["a"]
+      db.close
+
+      db = ScoutCabinet.open(tmpfile, false)
+      assert_equal "1", db["a"]
+    end
+  end
+end
+

data/test/scout/persist/test_tsv.rb

@@ -0,0 +1,146 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVPersist < Test::Unit::TestCase
+  def test_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      TmpFile.with_file(content) do |filename|
+        TSV.open(filename)
+      end
+    end
+
+    assert NamedArray === tsv["row1"]
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      TmpFile.with_file(content) do |filename|
+        TSV.open(filename)
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
+    assert_nothing_raised do
+      tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+        raise
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+
+  def test_persist_with_data
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = nil
+    TmpFile.with_file do |tk|
+      data = Persist.open_tokyocabinet(tk, true, "HDB")
+      assert Open.exists?(tk)
+      tsv = Persist.persist("TEST Persist TSV", :HDB, :persist_data => data) do |data|
+        t = TmpFile.with_file(content) do |filename|
+          TSV.open(filename, persist_data: data)
+        end
+        t
+        nil
+      end
+      refute Open.exists?(tk)
+      assert Open.exists?(data.persistence_path)
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
+    assert_nothing_raised do
+      tsv = Persist.persist("TEST Persist TSV", :HDB) do 
+        raise
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+
+  def test_tsv
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = nil
+    TmpFile.with_file(content) do |filename|
+      tsv = Persist.tsv("Some TSV") do |data|
+        TSV.open(filename, persist_data: data)
+      end
+      assert_equal ['b'], tsv["row1"][1]
+      assert NamedArray === tsv["row1"]
+      assert_equal ['b'], tsv["row1"]["ValueB"]
+      assert_include tsv.keys, 'row1'
+      assert_include tsv.keys, 'row2'
+      assert_nothing_raised do
+        tsv = Persist.tsv("Some TSV") do |data|
+          raise
+        end
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+
+  def test_persist_tsv
+    content =<<-'EOF'
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = nil
+    TmpFile.with_file(content) do |filename|
+      tsv = Persist.persist_tsv("Some TSV", sep: /\s+/, type: :double) do |data|
+        TSV.open(filename, sep: /\s+/, type: :double, persist_data: data)
+      end
+      assert_equal ['b'], tsv["row1"][1]
+      assert NamedArray === tsv["row1"]
+      assert_equal ['b'], tsv["row1"]["ValueB"]
+      assert_include tsv.keys, 'row1'
+      assert_include tsv.keys, 'row2'
+      assert_nothing_raised do
+        tsv = Persist.persist_tsv("Some TSV", sep: /\s+/, type: :double) do |data|
+          raise
+        end
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+end
+

data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb}

@@ -1,7 +1,6 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-require 'scout/tsv'
 class TestTSVAdapter < Test::Unit::TestCase
   def test_get_set
     tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
@@ -9,7 +8,6 @@ class TestTSVAdapter < Test::Unit::TestCase
     tsv.extend TSVAdapter
     tsv["a"] = %w(1 2 3)
 
-    assert_equal %w(1 2 3) * "\t", tsv.dup["a"]
     assert_equal %w(a), tsv.keys
     assert_equal [%w(1 2 3)], tsv.collect{|k,v| v }
     assert_equal [%w(1 2 3)], tsv.values
@@ -20,7 +18,6 @@ class TestTSVAdapter < Test::Unit::TestCase
     new.extend TSVAdapter
 
     tsv = new
-    assert_equal %w(1 2 3) * "\t", tsv.dup["a"]
     assert_equal %w(a), tsv.keys
     assert_equal [%w(1 2 3)], tsv.collect{|k,v| v }
     assert_equal [%w(1 2 3)], tsv.values
@@ -35,10 +32,12 @@ class TestTSVAdapter < Test::Unit::TestCase
     tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
     tsv.type = :list
     tsv.extend TSVAdapter
-    tsv.serializer = :integer_array
+    tsv.serializer = :marshal
     tsv["a"] = [1, 2, 3]
 
+
     assert_equal [1, 2, 3], tsv["a"]
+    assert_equal [1, 2, 3], Marshal.load(tsv.orig_get("a"))
   end
 end