scout-gear 7.2.0 → 8.0.0

data/test/scout/test_meta_extension.rb

@@ -8,6 +8,13 @@ class TestMetaExtension < Test::Unit::TestCase
     extension_attr :code, :code2
   end
 
+  module ExtensionClass2
+    extend MetaExtension
+
+    extension_attr :code3, :code4
+  end
+
+
   def test_setup_annotate
     str = "String"
     ExtensionClass.setup(str, :code)
@@ -51,5 +58,23 @@ class TestMetaExtension < Test::Unit::TestCase
     assert o.extension_attr_hash.include?(:code)
     assert o.extension_attr_hash.include?(:code2)
   end
+
+  def test_twice
+    str = "String"
+
+    ExtensionClass.setup(str, :code2 => :code)
+    assert_equal :code, str.code2
+    assert_include str.instance_variable_get(:@extension_attrs), :code
+
+    str.extend ExtensionClass2
+    str.code3 = :code_alt
+    assert_equal :code, str.code2
+    assert_equal :code_alt, str.code3
+    assert_include str.instance_variable_get(:@extension_attrs), :code
+    assert_include str.instance_variable_get(:@extension_attrs), :code3
+
+    assert_include str.extension_attr_hash, :code
+    assert_include str.extension_attr_hash, :code3
+  end
 end
 

data/test/scout/test_named_array.rb

@@ -15,5 +15,23 @@ ValueB (Entity type)
     assert_equal 1, NamedArray.identify_name(names, "ValueB")
     assert_equal 1, NamedArray.identify_name(names, 1)
   end
+
+  def test_missing_field
+    a = NamedArray.setup([1,2], [:a, :b])
+    assert_equal 1, a[:a]
+    assert_equal nil, a[:c]
+  end
+
+  def test_zip_fields
+    a = [%w(a b), %w(1 1)]
+    assert_equal [%w(a 1), %w(b 1)], NamedArray.zip_fields(a)
+  end
+
+  def test_add_zipped
+    a = [%w(a b), %w(1 1)]
+    NamedArray.add_zipped a, [%w(c), %w(1)]
+    NamedArray.add_zipped a, [%w(d), %w(1)]
+    assert_equal [%w(a b c d), %w(1 1 1 1)], a
+  end
 end
 

data/test/scout/test_persist.rb

@@ -95,6 +95,73 @@ class TestPersist < Test::Unit::TestCase
     end
   end
 
+  def test_concurrent
+    num = 10
+
+    s = 0.01
+    10.times do
+      TmpFile.with_file do |file|
+        output1 = file + '.output1'
+        output2 = file + '.output2'
+        pid1 = Process.fork do
+          Open.purge_pipes
+          sleep rand/10.0
+          io = Persist.persist("test", :string, :path => file) do
+            Open.open_pipe do |sin|
+              num.times do |i|
+                sin.puts "line-#{i}-#{Process.pid}"
+                sleep s
+              end
+            end
+          end
+          if IO === io
+            Open.consume_stream(io, false)
+          else
+            Open.write(output1, io)
+          end
+        end
+        pid2 = Process.fork do
+          Open.purge_pipes
+          sleep rand/10.0
+          io = Persist.persist("test", :string, :path => file) do
+            Open.open_pipe do |sin|
+              num.times do |i|
+                sin.puts "line-#{i}-#{Process.pid}"
+                sleep s
+              end
+            end
+          end
+          if IO === io
+            Open.consume_stream(io, false)
+          else
+            Open.write(output2, io)
+          end
+        end
+        Process.wait
+        Process.wait
+
+        assert File.exist?(output1) || File.exist?(output2)
+        [pid1, pid2].zip([output2, output1]).each do |pid, found|
+          next unless File.exist?(found)
+          assert Open.read(found).include? "-#{pid}\n"
+        end
+        [pid1, pid2].zip([output1, output2]).each do |pid, found|
+          next unless File.exist?(found)
+          refute Open.read(found).include? "-#{pid}\n"
+        end
+        Open.rm file
+        Open.rm output1
+        Open.rm output2
+      end
+    end
+  end
+
+  def test_path_prefix
+    Persist.persist('foo', :tsv, :prefix => "TSV") do |filename|
+      assert File.basename(filename).start_with? "TSV"
+    end
+  end
+
   def __test_speed
     times = 100_000
     TmpFile.with_file do |tmpfile|

data/test/scout/test_tmpfile.rb

@@ -10,7 +10,7 @@ class TestTmpFile < Test::Unit::TestCase
   def test_do_tmp_file
     content = "Hello World!"
     TmpFile.with_file(content) do |file|
-      assert_equal_path content, File.open(file).read
+      assert_equal content, File.open(file).read
     end
   end
 

data/test/scout/test_tsv.rb

@@ -22,7 +22,7 @@ row4    a    a    id3
     tsv = TmpFile.with_file(content) do |filename|
       TSV.open(filename, :persist => false)
     end
-
+    
     TmpFile.with_file(content2) do |filename|
       TSV.open(filename, :data => tsv)
     end
@@ -41,9 +41,14 @@ row2    a    a    id3
     EOF
 
     tsv = TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :persist => true)
+      tsv.close
+      Persist::CONNECTIONS.clear
       TSV.open(filename, :persist => true)
     end
 
+    assert_equal "Id", tsv.key_field
+
     assert tsv.respond_to?(:persistence_class)
     assert_equal TokyoCabinet::HDB, tsv.persistence_class
 
@@ -61,7 +66,15 @@ row2    a    a    id3
     EOF
 
     tsv = TmpFile.with_file(content) do |filename|
-      TSV.open(filename, :persist => true)
+      TSV.open(filename, :persist => false)
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+    assert_equal %w(A a), tsv["row2"][0]
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true, :merge => true)
     end
 
     assert tsv.respond_to?(:persistence_class)
@@ -69,6 +82,212 @@ row2    a    a    id3
 
     assert_include tsv.keys, 'row1'
     assert_include tsv.keys, 'row2'
+    assert_equal %w(A a), tsv["row2"][0]
+  end
+
+  def test_headerless_fields
+    content =<<-EOF
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :fields => [1])
+      assert_equal ["a", "aa", "aaa"], tsv["row1"][0]
+      assert_equal :double, tsv.type
+      assert_equal [%w(a aa aaa)], tsv["row1"]
+    end
+  end
+
+  def test_tsv_field_selection
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single)
+      assert_equal ["ValueA"], tsv.fields
+    end
+  end
+
+  def test_tsv_single_from_flat
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    1 2
+row2    4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value", :fields => ["Id"])
+      assert_equal "row1", tsv["1"]
+    end
+  end
+
+  def test_key_field
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value")
+      assert_equal %w(Id), tsv.fields
+      assert_equal "Value", tsv.key_field
+      assert_equal "a", tsv["1"]
+    end
+  end
+
+  def test_fix
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :fix => Proc.new{|l| if l =~ /1/;then "a 3" else l end})
+      assert_equal "a", tsv["3"]
+    end
+  end
+
+  def test_flat
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    a|aa|aaa
+row2    A|AA|AAA
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat).include? "row1"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat)["row1"].include? "a"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id")["row1"].include? "a"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id", :fields => ["Value"])["row1"].include? "a"
+    end
   end
-end
 
+  def test_tsv_flat_double
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "ValueA", :fields => ["OtherID"], :merge => true)
+      assert tsv["aaa"].include? "Id1"
+      assert tsv["aaa"].include? "Id2"
+    end
+  end
+
+  def test_flat2single
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    a aa aaa
+row2    A AA AAA
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value")
+      assert tsv.include? "aaa"
+    end
+  end
+
+  def test_flat_key
+    content =<<-EOF
+#Id    ValueA 
+row1   a   aa   aaa
+row2   b  bbb bbbb bb aa
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :key_field => "ValueA")
+      assert_equal ["row1"], tsv["a"]
+      assert_equal ["row1", "row2"], tsv["aa"]
+    end
+  end
+
+  def test_unnamed_key
+    content =<<-EOF
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :key_field => 1)
+      assert tsv.keys.include? "a"
+    end
+  end
+
+  def test_grep
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+c 3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :grep => "#\\|2")
+      assert_includes tsv, "2"
+      refute_includes tsv, "3"
+    end
+  end
+
+  def TODO_test_tsv_grep
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+b 3
+d 22
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :tsv_grep => "2")
+      assert_includes tsv, "2"
+      refute_includes tsv, "3"
+      assert(tsv.include?("2"))
+      assert(! tsv.include?("3"))
+    end
+  end
+
+  def test_flat_with_field_header
+    content =<<-EOF
+#: :type=:flat
+#Id    ValueA
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => false)
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+  end
+
+  def test_alt_args
+    content =<<-EOF
+#Id    ValueA
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, type: :flat, :sep => /\s+/, :merge => false)
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+
+    tsv = TSV.str_setup("ID~ValueA,ValueB#:type=:flat", {})
+    assert_equal "ID", tsv.key_field
+  end
+end

data/test/scout/test_work_queue.rb

@@ -103,18 +103,19 @@ class TestWorkQueue < Test::Unit::TestCase
       res << out
     end
 
-    pid = Process.fork do
-      reps.times do |i|
-        q.write i
+    Log.with_severity 7 do
+      pid = Process.fork do
+        reps.times do |i|
+          q.write i
+        end
       end
-    end
 
-    Process.waitpid pid
-    q.close
+      Process.waitpid pid
 
-    assert_raise ScoutException do
-      q.join
-      t.join
+      assert_raise ScoutException do
+        q.join
+        t.join
+      end
     end
   end
 
@@ -128,20 +129,22 @@ class TestWorkQueue < Test::Unit::TestCase
 
     res = []
     q.process do |out|
-      raise ScoutException if rand < 0.01
+      raise ScoutException 
       res << out
     end
 
-    pid = Process.fork do
-      reps.times do |i|
-        q.write i
+    Log.with_severity 7 do
+      pid = Process.fork do
+        reps.times do |i|
+          q.write i
+        end
+        q.close
       end
-      q.close
-    end
 
-    assert_raise ScoutException do
-      q.join
-      t.join
+      assert_raise ScoutException do
+        q.join
+        t.join
+      end
     end
   end
 end

data/test/scout/tsv/persist/test_adapter.rb

@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/tsv'
 class TestTSVAdapter < Test::Unit::TestCase
-  def test_get_set
+  def _test_get_set
     tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
     tsv.type = :list
     tsv.extend TSVAdapter
@@ -30,5 +30,15 @@ class TestTSVAdapter < Test::Unit::TestCase
     assert_equal [["a", %w(1 2 3)], ["b", %w(11 22 33)]], tsv.sort
     assert_equal [["b", %w(11 22 33)], ["a", %w(1 2 3)]], tsv.sort_by{|k,v| -v[0].to_i }
   end
+
+  def test_serializer
+    tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
+    tsv.type = :list
+    tsv.extend TSVAdapter
+    tsv.serializer = :integer_array
+    tsv["a"] = [1, 2, 3]
+
+    assert_equal [1, 2, 3], tsv["a"]
+  end
 end
 

data/test/scout/tsv/persist/test_fix_width_table.rb

@@ -0,0 +1,134 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestFixWidthTable < Test::Unit::TestCase
+  def load_data(data)
+    tsv = TSV.open(data, type: :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
+    tsv.add_field "Start" do |key, values|
+      values["Range"].first
+    end
+    tsv.add_field "End" do |key, values|
+      values["Range"].last
+    end
+
+    tsv = tsv.slice ["Start", "End"]
+
+    tsv
+  end
+
+  def test_options
+    TmpFile.with_file do |filename|
+      f = FixWidthTable.new filename, 100, true
+      f.close
+
+      f1 = FixWidthTable.new filename, 100, false
+
+      assert_equal true, f1.range
+    end
+  end
+
+  def test_add
+    TmpFile.with_file do |filename|
+      f = FixWidthTable.new filename, 100, true
+      f.add [1,2,0], "test1"
+      f.add [3,4,0], "test2"
+      f.read
+
+      assert_equal 1, f.idx_pos(0)
+      assert_equal 3, f.idx_pos(1)
+      assert_equal 2, f.idx_pos_end(0)
+      assert_equal 4, f.idx_pos_end(1)
+      assert_equal 0, f.idx_overlap(0)
+      assert_equal 0, f.idx_overlap(1)
+      assert_equal "test1", f.idx_value(0)
+      assert_equal "test2", f.idx_value(1)
+
+    end
+
+  end
+
+  def test_point
+    data =<<-EOF
+#: :sep=/\\s+/#:type=:single#:cast=:to_i
+#ID Pos
+a 1
+b 10
+c 20
+d 12
+e 26
+f 11
+g 25
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = TSV.open datafile
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, false
+        f.add_point tsv
+        f.read
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[10].sort
+        assert_equal %w(a b c d f), f[(0..20)].sort
+      end
+    end
+  end
+
+  def test_range
+    data =<<-EOF
+##012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[1].sort
+        assert_equal %w(), f[20].sort
+        assert_equal %w(), f[(20..100)].sort
+        assert_equal %w(a b d), f[3].sort
+        assert_equal %w(a b c d e), f[(3..4)].sort
+        assert_equal %w(a c e), f[7].sort
+      end
+    end
+  end
+
+
+  def test_range_pos
+    data =<<-EOF
+##012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f.overlaps(0).sort
+        assert_equal %w(1:6), f.overlaps(1).sort
+        assert_equal %w(1:6:b), f.overlaps(1, true).sort
+      end
+    end
+  end
+end
+

data/test/scout/tsv/persist/test_tokyocabinet.rb

@@ -21,6 +21,11 @@ row2    a    a    id3
 
     assert_equal %w(a aa aaa), tsv["row1"][0]
 
+    assert TSVAdapter === tsv
+    assert TSV === tsv
+    assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
+    assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
+
     tsv_loaded = assert_nothing_raised do
       TmpFile.with_file(content) do |filename|
         Persist.persist(__method__, :HDB) do
@@ -29,6 +34,7 @@ row2    a    a    id3
       end
     end
 
+
     assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
   end
 
@@ -54,7 +60,7 @@ row2    a    a    id3
     end
   end
 
-  def test_speed
+  def __test_speed
     tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
 
     size = 100_000
@@ -88,5 +94,27 @@ row2    a    a    id3
       assert_equal tc["key-#{i}"], tsv["key-#{i}"]
     end
   end
+
+  def test_float_array
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   0.2   0.3 0
+row2    0.1  4.5 0
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f)
+      tsv.save_extension_attr_hash
+      assert_equal [0.2, 0.3, 0], tsv["row1"]
+      assert_equal TSVAdapter::FloatArraySerializer, tsv.serializer
+      Open.cp tsv.persistence_path, tmpdir.persistence.foo
+      tsv2 = ScoutCabinet.open(tmpdir.persistence.foo, false)
+      tsv2.extend TSVAdapter
+      assert_equal [0.2, 0.3, 0], tsv2["row1"]
+      assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
+    end
+ 
+  end
+
 end