scout-gear 7.3.0 → 8.1.0

data/test/scout/test_tsv.rb

@@ -22,7 +22,7 @@ row4    a    a    id3
     tsv = TmpFile.with_file(content) do |filename|
       TSV.open(filename, :persist => false)
     end
-
+    
     TmpFile.with_file(content2) do |filename|
       TSV.open(filename, :data => tsv)
     end
@@ -41,9 +41,14 @@ row2    a    a    id3
     EOF
 
     tsv = TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :persist => true)
+      tsv.close
+      Persist::CONNECTIONS.clear
       TSV.open(filename, :persist => true)
     end
 
+    assert_equal "Id", tsv.key_field
+
     assert tsv.respond_to?(:persistence_class)
     assert_equal TokyoCabinet::HDB, tsv.persistence_class
 
@@ -79,5 +84,227 @@ row2    a    a    id3
     assert_include tsv.keys, 'row2'
     assert_equal %w(A a), tsv["row2"][0]
   end
-end
 
+  def test_headerless_fields
+    content =<<-EOF
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :fields => [1])
+      assert_equal ["a", "aa", "aaa"], tsv["row1"][0]
+      assert_equal :double, tsv.type
+      assert_equal [%w(a aa aaa)], tsv["row1"]
+    end
+  end
+
+  def test_tsv_field_selection
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single)
+      assert_equal ["ValueA"], tsv.fields
+    end
+  end
+
+  def test_tsv_single_from_flat
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    1 2
+row2    4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value", :fields => ["Id"])
+      assert_equal "row1", tsv["1"]
+    end
+  end
+
+  def test_key_field
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value")
+      assert_equal %w(Id), tsv.fields
+      assert_equal "Value", tsv.key_field
+      assert_equal "a", tsv["1"]
+    end
+  end
+
+  def test_fix
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :fix => Proc.new{|l| if l =~ /1/;then "a 3" else l end})
+      assert_equal "a", tsv["3"]
+    end
+  end
+
+  def test_flat
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    a|aa|aaa
+row2    A|AA|AAA
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat).include? "row1"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat)["row1"].include? "a"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id")["row1"].include? "a"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id", :fields => ["Value"])["row1"].include? "a"
+    end
+  end
+
+  def test_tsv_flat_double
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "ValueA", :fields => ["OtherID"], :merge => true)
+      assert tsv["aaa"].include? "Id1"
+      assert tsv["aaa"].include? "Id2"
+    end
+  end
+
+  def test_flat2single
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    a aa aaa
+row2    A AA AAA
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value")
+      assert tsv.include? "aaa"
+    end
+  end
+
+  def test_flat_key
+    content =<<-EOF
+#Id    ValueA 
+row1   a   aa   aaa
+row2   b  bbb bbbb bb aa
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :key_field => "ValueA")
+      assert_equal ["row1"], tsv["a"]
+      assert_equal ["row1", "row2"], tsv["aa"]
+    end
+  end
+
+  def test_unnamed_key
+    content =<<-EOF
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :key_field => 1)
+      assert tsv.keys.include? "a"
+    end
+  end
+
+  def test_grep
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+c 3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :grep => "#\\|2")
+      assert_includes tsv, "2"
+      refute_includes tsv, "3"
+    end
+  end
+
+  def TODO_test_tsv_grep
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+b 3
+d 22
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :tsv_grep => "2")
+      assert_includes tsv, "2"
+      refute_includes tsv, "3"
+      assert(tsv.include?("2"))
+      assert(! tsv.include?("3"))
+    end
+  end
+
+  def test_flat_with_field_header
+    content =<<-EOF
+#: :type=:flat
+#Id    ValueA
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => false)
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+  end
+
+  def test_alt_args
+    content =<<-EOF
+#Id    ValueA
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, type: :flat, :sep => /\s+/, :merge => false)
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+
+    tsv = TSV.str_setup("ID~ValueA,ValueB#:type=:flat", {})
+    assert_equal "ID", tsv.key_field
+  end
+
+  def test_cast_in_header
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+c 3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :grep => "#\\|2")
+      refute tsv.to_s.include?(":cast=:to_f")
+      tsv.cast = :to_f
+      assert_include tsv.to_s, ":cast=:to_f"
+    end
+  end
+end

data/test/scout/test_work_queue.rb

@@ -90,59 +90,83 @@ class TestWorkQueue < Test::Unit::TestCase
   end
 
   def test_queue_error
-    num = 100
-    reps = 10_000
-
-    sss 0
-    q = WorkQueue.new num do |obj|
-      raise ScoutException if rand < 0.1
-      [Process.pid.to_s, obj.to_s] * " "
-    end
-
-    res = []
-    q.process do |out|
-      res << out
-    end
+    5.times do |i|
+      num = 100
+      reps = 10_000
 
-    pid = Process.fork do
-      reps.times do |i|
-        q.write i
+      q = WorkQueue.new num do |obj|
+        raise ScoutException if rand < 0.1
+        [Process.pid.to_s, obj.to_s] * " "
       end
-    end
 
-    Process.waitpid pid
-    q.close
+      res = []
+      q.process do |out|
+        res << out
+      end
 
-    assert_raise ScoutException do
-      q.join
-      t.join
+      Log.with_severity 7 do
+        t = Thread.new do
+          Thread.current["name"] = "queue writer"
+          Thread.current.report_on_exception = false
+          reps.times do |i|
+            q.write i
+          end
+          q.close
+        end
+        Thread.pass until t["name"]
+
+        assert_raise ScoutException do
+          begin
+            q.join(false)
+          rescue
+            t.raise($!)
+            raise $!
+          ensure
+            t.join
+          end
+        end
+      end
     end
   end
 
   def test_queue_error_in_input
-    num = 100
-    reps = 10_000
-
-    q = WorkQueue.new num do |obj|
-      [Process.pid.to_s, obj.to_s] * " "
-    end
+    5.times do |i|
+      num = 100
+      reps = 10_000
 
-    res = []
-    q.process do |out|
-      raise ScoutException if rand < 0.01
-      res << out
-    end
+      q = WorkQueue.new num do |obj|
+        [Process.pid.to_s, obj.to_s] * " "
+      end
 
-    pid = Process.fork do
-      reps.times do |i|
-        q.write i
+      res = []
+      q.process do |out|
+        raise ScoutException 
+        res << out
       end
-      q.close
-    end
 
-    assert_raise ScoutException do
-      q.join
-      t.join
+      Log.with_severity 7 do
+        t = Thread.new do
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "queue writer"
+          reps.times do |i|
+            q.write i
+          end
+          q.close
+        end
+        Thread.pass until t["name"]
+
+        assert_raise ScoutException do
+          begin
+            q.join(false)
+          rescue Exception
+            t.raise($!)
+            raise $!
+          ensure
+            t.join
+            q.clean
+          end
+        end
+      end
     end
   end
 end

data/test/scout/tsv/persist/test_tokyocabinet.rb

@@ -21,6 +21,11 @@ row2    a    a    id3
 
     assert_equal %w(a aa aaa), tsv["row1"][0]
 
+    assert TSVAdapter === tsv
+    assert TSV === tsv
+    assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
+    assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
+
     tsv_loaded = assert_nothing_raised do
       TmpFile.with_file(content) do |filename|
         Persist.persist(__method__, :HDB) do
@@ -29,6 +34,7 @@ row2    a    a    id3
       end
     end
 
+
     assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
   end
 
@@ -54,7 +60,7 @@ row2    a    a    id3
     end
   end
 
-  def test_speed
+  def __test_speed
     tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
 
     size = 100_000
@@ -88,5 +94,27 @@ row2    a    a    id3
       assert_equal tc["key-#{i}"], tsv["key-#{i}"]
     end
   end
+
+  def test_float_array
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   0.2   0.3 0
+row2    0.1  4.5 0
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f)
+      tsv.save_extension_attr_hash
+      assert_equal [0.2, 0.3, 0], tsv["row1"]
+      assert_equal TSVAdapter::FloatArraySerializer, tsv.serializer
+      Open.cp tsv.persistence_path, tmpdir.persistence.foo
+      tsv2 = ScoutCabinet.open(tmpdir.persistence.foo, false)
+      tsv2.extend TSVAdapter
+      assert_equal [0.2, 0.3, 0], tsv2["row1"]
+      assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
+    end
+ 
+  end
+
 end
 

data/test/scout/tsv/test_attach.rb

@@ -0,0 +1,227 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVAttach < Test::Unit::TestCase
+  def test_attach_simple
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, :complete => true
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+        assert_equal %w(Id3), tsv["row3"]["OtherID"]
+        assert_equal %w(B), tsv["row3"]["ValueB"]
+      end
+    end
+  end
+
+  def test_attach_by_key
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    B1|B11    Id1|Id11
+row2.2    B2|B22|B222    Id2.2|Id22.2|Id222.2
+row3    B3    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, complete: true, match_key: "ValueB"
+        assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
+        assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
+        assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
+        assert_equal %w(Id2.2 Id22.2), tsv["row2"]["OtherID"]
+      end
+    end
+  end
+
+  def test_attach_by_reorder
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ValueB ID  OtherID
+B1  row1|row1.1  Id1|Id11
+B2 row2    Id2.2|Id22.2|Id222.2
+B3 row3   Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, match_key: "ID", one2one: false
+        assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
+        assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
+        assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
+        assert_equal %w(Id2.2 Id22.2 Id222.2), tsv["row2"]["OtherID"]
+      end
+    end
+  end
+
+
+  def test_attach_same_key
+    content1 =<<-EOF
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, fields: "OtherID"
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
+
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, fields: "OtherID"
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal "Id1", tsv1["row1"]["OtherID"]
+  end
+
+  def test_attach_source_field
+    content1 =<<-EOF
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#ValueB    OtherID
+b    Id1|Id2
+B    Id3
+    EOF
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, bar: true
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
+
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal "Id1", tsv1["row1"]["OtherID"]
+  end
+
+  def test_attach_transformer
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        out = TSV.attach filename1, filename2, target: :stream, bar: false
+        tsv = out.tsv
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+      end
+    end
+  end
+
+  def test_attach_flexible_names
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#Identifiers(ID) OtherID
+row1        Id1|Id2
+row3        Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        out = TSV.attach filename1, filename2, target: :stream, bar: false
+        tsv = out.tsv
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+      end
+    end
+  end
+end