scout-gear 7.3.0 → 8.0.0

data/test/scout/test_tsv.rb

@@ -22,7 +22,7 @@ row4    a    a    id3
     tsv = TmpFile.with_file(content) do |filename|
       TSV.open(filename, :persist => false)
     end
-
+    
     TmpFile.with_file(content2) do |filename|
       TSV.open(filename, :data => tsv)
     end
@@ -41,9 +41,14 @@ row2    a    a    id3
     EOF
 
     tsv = TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :persist => true)
+      tsv.close
+      Persist::CONNECTIONS.clear
       TSV.open(filename, :persist => true)
     end
 
+    assert_equal "Id", tsv.key_field
+
     assert tsv.respond_to?(:persistence_class)
     assert_equal TokyoCabinet::HDB, tsv.persistence_class
 
@@ -79,5 +84,210 @@ row2    a    a    id3
     assert_include tsv.keys, 'row2'
     assert_equal %w(A a), tsv["row2"][0]
   end
-end
 
+  def test_headerless_fields
+    content =<<-EOF
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :fields => [1])
+      assert_equal ["a", "aa", "aaa"], tsv["row1"][0]
+      assert_equal :double, tsv.type
+      assert_equal [%w(a aa aaa)], tsv["row1"]
+    end
+  end
+
+  def test_tsv_field_selection
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single)
+      assert_equal ["ValueA"], tsv.fields
+    end
+  end
+
+  def test_tsv_single_from_flat
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    1 2
+row2    4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value", :fields => ["Id"])
+      assert_equal "row1", tsv["1"]
+    end
+  end
+
+  def test_key_field
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value")
+      assert_equal %w(Id), tsv.fields
+      assert_equal "Value", tsv.key_field
+      assert_equal "a", tsv["1"]
+    end
+  end
+
+  def test_fix
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :fix => Proc.new{|l| if l =~ /1/;then "a 3" else l end})
+      assert_equal "a", tsv["3"]
+    end
+  end
+
+  def test_flat
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    a|aa|aaa
+row2    A|AA|AAA
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat).include? "row1"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat)["row1"].include? "a"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id")["row1"].include? "a"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id", :fields => ["Value"])["row1"].include? "a"
+    end
+  end
+
+  def test_tsv_flat_double
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "ValueA", :fields => ["OtherID"], :merge => true)
+      assert tsv["aaa"].include? "Id1"
+      assert tsv["aaa"].include? "Id2"
+    end
+  end
+
+  def test_flat2single
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    a aa aaa
+row2    A AA AAA
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value")
+      assert tsv.include? "aaa"
+    end
+  end
+
+  def test_flat_key
+    content =<<-EOF
+#Id    ValueA 
+row1   a   aa   aaa
+row2   b  bbb bbbb bb aa
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :key_field => "ValueA")
+      assert_equal ["row1"], tsv["a"]
+      assert_equal ["row1", "row2"], tsv["aa"]
+    end
+  end
+
+  def test_unnamed_key
+    content =<<-EOF
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :key_field => 1)
+      assert tsv.keys.include? "a"
+    end
+  end
+
+  def test_grep
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+c 3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :grep => "#\\|2")
+      assert_includes tsv, "2"
+      refute_includes tsv, "3"
+    end
+  end
+
+  def TODO_test_tsv_grep
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+b 3
+d 22
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :tsv_grep => "2")
+      assert_includes tsv, "2"
+      refute_includes tsv, "3"
+      assert(tsv.include?("2"))
+      assert(! tsv.include?("3"))
+    end
+  end
+
+  def test_flat_with_field_header
+    content =<<-EOF
+#: :type=:flat
+#Id    ValueA
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => false)
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+  end
+
+  def test_alt_args
+    content =<<-EOF
+#Id    ValueA
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, type: :flat, :sep => /\s+/, :merge => false)
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+
+    tsv = TSV.str_setup("ID~ValueA,ValueB#:type=:flat", {})
+    assert_equal "ID", tsv.key_field
+  end
+end

data/test/scout/test_work_queue.rb

@@ -93,7 +93,6 @@ class TestWorkQueue < Test::Unit::TestCase
     num = 100
     reps = 10_000
 
-    sss 0
     q = WorkQueue.new num do |obj|
       raise ScoutException if rand < 0.1
       [Process.pid.to_s, obj.to_s] * " "
@@ -104,18 +103,19 @@ class TestWorkQueue < Test::Unit::TestCase
       res << out
     end
 
-    pid = Process.fork do
-      reps.times do |i|
-        q.write i
+    Log.with_severity 7 do
+      pid = Process.fork do
+        reps.times do |i|
+          q.write i
+        end
       end
-    end
 
-    Process.waitpid pid
-    q.close
+      Process.waitpid pid
 
-    assert_raise ScoutException do
-      q.join
-      t.join
+      assert_raise ScoutException do
+        q.join
+        t.join
+      end
     end
   end
 
@@ -129,20 +129,22 @@ class TestWorkQueue < Test::Unit::TestCase
 
     res = []
     q.process do |out|
-      raise ScoutException if rand < 0.01
+      raise ScoutException 
       res << out
     end
 
-    pid = Process.fork do
-      reps.times do |i|
-        q.write i
+    Log.with_severity 7 do
+      pid = Process.fork do
+        reps.times do |i|
+          q.write i
+        end
+        q.close
       end
-      q.close
-    end
 
-    assert_raise ScoutException do
-      q.join
-      t.join
+      assert_raise ScoutException do
+        q.join
+        t.join
+      end
     end
   end
 end

data/test/scout/tsv/persist/test_adapter.rb

@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/tsv'
 class TestTSVAdapter < Test::Unit::TestCase
-  def test_get_set
+  def _test_get_set
     tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
     tsv.type = :list
     tsv.extend TSVAdapter

data/test/scout/tsv/persist/test_tokyocabinet.rb

@@ -21,6 +21,11 @@ row2    a    a    id3
 
     assert_equal %w(a aa aaa), tsv["row1"][0]
 
+    assert TSVAdapter === tsv
+    assert TSV === tsv
+    assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
+    assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
+
     tsv_loaded = assert_nothing_raised do
       TmpFile.with_file(content) do |filename|
         Persist.persist(__method__, :HDB) do
@@ -29,6 +34,7 @@ row2    a    a    id3
       end
     end
 
+
     assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
   end
 
@@ -54,7 +60,7 @@ row2    a    a    id3
     end
   end
 
-  def test_speed
+  def __test_speed
     tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
 
     size = 100_000
@@ -88,5 +94,27 @@ row2    a    a    id3
       assert_equal tc["key-#{i}"], tsv["key-#{i}"]
     end
   end
+
+  def test_float_array
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   0.2   0.3 0
+row2    0.1  4.5 0
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f)
+      tsv.save_extension_attr_hash
+      assert_equal [0.2, 0.3, 0], tsv["row1"]
+      assert_equal TSVAdapter::FloatArraySerializer, tsv.serializer
+      Open.cp tsv.persistence_path, tmpdir.persistence.foo
+      tsv2 = ScoutCabinet.open(tmpdir.persistence.foo, false)
+      tsv2.extend TSVAdapter
+      assert_equal [0.2, 0.3, 0], tsv2["row1"]
+      assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
+    end
+ 
+  end
+
 end
 

data/test/scout/tsv/test_attach.rb

@@ -0,0 +1,227 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVAttach < Test::Unit::TestCase
+  def test_attach_simple
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, :complete => true
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+        assert_equal %w(Id3), tsv["row3"]["OtherID"]
+        assert_equal %w(B), tsv["row3"]["ValueB"]
+      end
+    end
+  end
+
+  def test_attach_by_key
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    B1|B11    Id1|Id11
+row2.2    B2|B22|B222    Id2.2|Id22.2|Id222.2
+row3    B3    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, complete: true, match_key: "ValueB"
+        assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
+        assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
+        assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
+        assert_equal %w(Id2.2 Id22.2), tsv["row2"]["OtherID"]
+      end
+    end
+  end
+
+  def test_attach_by_reorder
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ValueB ID  OtherID
+B1  row1|row1.1  Id1|Id11
+B2 row2    Id2.2|Id22.2|Id222.2
+B3 row3   Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, match_key: "ID", one2one: false
+        assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
+        assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
+        assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
+        assert_equal %w(Id2.2 Id22.2 Id222.2), tsv["row2"]["OtherID"]
+      end
+    end
+  end
+
+
+  def test_attach_same_key
+    content1 =<<-EOF
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, fields: "OtherID"
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
+
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, fields: "OtherID"
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal "Id1", tsv1["row1"]["OtherID"]
+  end
+
+  def test_attach_source_field
+    content1 =<<-EOF
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#ValueB    OtherID
+b    Id1|Id2
+B    Id3
+    EOF
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, bar: true
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
+
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal "Id1", tsv1["row1"]["OtherID"]
+  end
+
+  def test_attach_transformer
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        out = TSV.attach filename1, filename2, target: :stream, bar: false
+        tsv = out.tsv
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+      end
+    end
+  end
+
+  def test_attach_flexible_names
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#Identifiers(ID) OtherID
+row1        Id1|Id2
+row3        Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        out = TSV.attach filename1, filename2, target: :stream, bar: false
+        tsv = out.tsv
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+      end
+    end
+  end
+end

data/test/scout/tsv/test_change_id.rb

@@ -0,0 +1,98 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestChangeID < Test::Unit::TestCase
+  def test_simple_reorder
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TSV.open StringIO.new(content1)
+
+    res = tsv.change_key "ValueA", keep: true
+    assert_equal ["row1"], res["A1"]["ID"]
+    assert_equal ["row1"], res["A11"]["ID"]
+    assert_equal ["row2"], res["A2"]["ID"]
+
+    res = tsv.change_key "ValueA", keep: false, one2one: true
+    assert_equal ["B1"], res["A1"]["ValueB"]
+    assert_equal ["B11"], res["A11"]["ValueB"]
+  end
+
+  def test_simple_reorder_file
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    TmpFile.with_file(content1) do |file1|
+      res = TSV.change_key file1, "ValueA", keep: true
+      assert_equal ["row1"], res["A1"]["ID"]
+      assert_equal ["row1"], res["A11"]["ID"]
+      assert_equal ["row2"], res["A2"]["ID"]
+      assert_equal ["B1","B11"], res["A1"]["ValueB"]
+
+      res = TSV.change_key file1, "ValueA", one2one: true, keep: true
+      assert_equal ["row1"], res["A1"]["ID"]
+      assert_equal ["row1"], res["A11"]["ID"]
+      assert_equal ["row2"], res["A2"]["ID"]
+      assert_equal ["B1"], res["A1"]["ValueB"]
+    end
+  end
+
+  def test_change_key_identifiers
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    identifiers_content =<<-EOF
+#: :sep=" "
+#ID    ValueC    ValueD
+row1    C1|C11    D1|D11
+row2    C2|C22    D2|D22
+    EOF
+
+
+    tsv = TSV.open StringIO.new(content1)
+    identifiers = TSV.open StringIO.new(identifiers_content)
+
+    res = tsv.change_key "ValueC", identifiers: identifiers, keep: true
+    assert_equal ["row1"], res["C1"]["ID"]
+    assert_equal ["row1"], res["C11"]["ID"]
+    assert_equal ["row2"], res["C2"]["ID"]
+  end
+
+  def test_change_id_identifiers
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    identifiers_content =<<-EOF
+#: :sep=" "
+#ID    ValueC    ValueD
+row1    C1|C11    D1|D11
+row2    C2|C22    D2|D22
+    EOF
+
+
+    tsv = TSV.open StringIO.new(content1)
+    identifiers = TSV.open StringIO.new(identifiers_content)
+
+    res = tsv.change_id "ValueA", "ValueC", identifiers: identifiers
+    assert_equal ["C1","C11"], res["row1"]["ValueC"]
+    assert_equal ["C2","C22"], res["row2"]["ValueC"]
+  end
+end
+

data/test/scout/tsv/test_dumper.rb

@@ -37,7 +37,7 @@ a\t1|11\t2|22
     end
 
     assert_raise ScoutException do
-      dumper.stream.read
+      TSV.open(dumper.stream, bar: true)
     end
   end
 end