scout-gear 7.2.0 → 8.0.0

data/test/scout/tsv/test_parser.rb

@@ -29,15 +29,16 @@ class TestTSVParser < Test::Unit::TestCase
     assert_equal (1..10).collect{|v| [v,v] }, values
   end
 
-  def __test_benchmark
+  def ___test_benchmark
     num = 10_000
     txt = num.times.inject(nil) do |acc,i|
       (acc.nil? ? "" : acc << "\n") << (0..10).collect{|v| v == 0 ? i : [v,v] * "|" } * "\t"
     end 
 
     txt = StringIO.new(([txt] * (10))*"\n")
-    Misc.benchmark 1 do
-    #Misc.profile do
+    Misc.benchmark 5 do
+      txt.rewind
+      #Misc.profile do
       data = TSV.parse_stream(txt, fix: true, type: :double, bar: true, merge: :concat)
       assert_equal num, data.size
       assert_equal 20, data['1'][0].length
@@ -170,4 +171,45 @@ k a|A b|B
 
     assert_equal [["k", [%w(b B)]]], values
   end
+
+  def test_parser_traverse_all
+    content =<<-EOF
+Key ValueA ValueB
+k a|A b|B
+    EOF
+    content = StringIO.new content
+
+    parser = TSV::Parser.new content, sep: " ", header_hash: ''
+
+    assert_equal "Key", parser.key_field
+
+    values = []
+    parser.traverse key_field: "ValueA", fields: :all, type: :double do |k,v|
+      values << v
+    end
+
+    assert_include values.flatten, 'a'
+  end
+
+
+  def test_parse_persist_serializer
+    content =<<-EOF
+Key ValueA ValueB
+k 1 2
+    EOF
+    content = StringIO.new content
+
+    TmpFile.with_file do |db|
+      data = ScoutCabinet.open db, true, "HDB"
+      TSV.parse content, sep: " ", header_hash: '', data: data, cast: :to_i, type: :list
+      assert_equal [1, 2], data["k"]
+    end
+
+    TmpFile.with_file do |db|
+      content.rewind
+      data = ScoutCabinet.open db, true, "HDB"
+      TSV.parse content, sep: " ", header_hash: '', data: data, cast: :to_i, type: :list, serializer: :float_array
+      assert_equal [1.0, 2.0], data["k"]
+    end
+  end
 end

data/test/scout/tsv/test_persist.rb

@@ -14,6 +14,15 @@ row2    a    a    id3
     EOF
 
 
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      TmpFile.with_file(content) do |filename|
+        TSV.open(filename)
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
     tsv = Persist.persist("TEST Persist TSV", :tsv) do 
       TmpFile.with_file(content) do |filename|
         TSV.open(filename)

data/test/scout/tsv/test_transformer.rb

@@ -0,0 +1,108 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVTransformer < Test::Unit::TestCase
+  def test_traverse
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    TmpFile.with_file(content) do |file|
+      parser = TSV::Parser.new file
+      dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Values"], type: :flat
+      dumper.init
+
+      trans = TSV::Transformer.new parser, dumper
+      dumper = trans.traverse do |k,values|
+        [k, values.flatten]
+      end
+
+      tsv = trans.tsv
+      assert_equal %w(A1 A11 B1 B11), tsv['row1']
+    end
+  end
+
+  def test_each
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    TmpFile.with_file(content) do |file|
+      parser = TSV::Parser.new file
+      dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Values"], type: :flat
+      dumper.init
+
+      trans = TSV::Transformer.new parser, dumper
+      dumper = trans.each do |k,values|
+        values.replace values.flatten
+      end
+
+      trans["row3"] = %w(A3 A33)
+
+      tsv = trans.tsv
+      assert_equal %w(A1 A11 B1 B11), tsv['row1']
+      assert_equal %w(A3 A33), tsv['row3']
+    end
+  end
+
+  def test_no_dumper_no_parser
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    TmpFile.with_file(content) do |file|
+      trans = TSV::Transformer.new file
+      trans.key_field = "Key"
+      trans.fields = ["Values"]
+      trans.type = :flat
+      trans.sep = "\t"
+
+
+      trans["row3"] = %w(A3 A33)
+
+      trans.each do |k,values|
+        values.replace values.flatten
+      end
+
+      tsv = trans.tsv
+      assert_equal %w(A1 A11 B1 B11), tsv['row1']
+      assert_equal %w(A3 A33), tsv['row3']
+    end
+  end
+
+  def test_to_list
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TSV.open(content)
+    assert_equal "A1", tsv.to_list["row1"]["ValueA"]
+    assert_equal "B2", tsv.to_list["row2"]["ValueB"]
+  end
+
+  def test_to_single
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TSV.open(content)
+    assert_equal "A1", tsv.to_single["row1"]
+    assert_equal "A2", tsv.to_single["row2"]
+  end
+end

data/test/scout/tsv/test_traverse.rb

@@ -1,9 +1,201 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-class TestClass < Test::Unit::TestCase
-  def test_true
-    assert true
+require 'scout/tsv'
+
+class TestTSVTraverse < Test::Unit::TestCase
+  def test_tsv_traverse_double
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB) do |k,v|
+      res[k] = v
+    end
+    assert_equal [%w(Id3 Id33), %w(B BB)], res["row2"]
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB), type: :list do |k,v|
+      res[k] = v
+    end
+    assert_equal ["Id3", "B"], res["row2"]
+
+    res = {}
+    tsv.traverse "OtherID", %w(Id ValueB), one2one: :strict do |k,v|
+      res[k] = v
+    end
+    assert_equal [[nil], %w(BB)], res["Id33"]
+
+    res = {}
+    tsv.traverse "OtherID", %w(Id ValueB), one2one: :strict, type: :list do |k,v|
+      res[k] = v
+    end
+    assert_equal ["row2", "B"], res["Id3"]
+    assert_equal [nil, "BB"], res["Id33"]
+
+    res = {}
+    tsv.traverse "OtherID", %w(Id ValueB), one2one: true, type: :list do |k,v|
+      res[k] = v
+    end
+    assert_equal ["row2", "B"], res["Id3"]
+    assert_equal ["row2", "BB"], res["Id33"]
+
+
+    tsv.traverse "OtherID", %w(Id ValueB), one2one: false, type: :list do |k,v|
+      res[k] = v
+    end
+    assert_equal ["row2", "B"], res["Id3"]
+    assert_equal [nil, "BB"], res["Id33"]
+
+    res = {}
+    key_name, field_names = tsv.traverse "OtherID" do |k,v|
+      res[k] = v
+    end
+    assert_equal "OtherID", key_name
+    assert_equal %w(Id ValueA ValueB), field_names
+  end
+
+  def test_tsv_traverse_all
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    all_values = []
+    tsv.traverse "ValueA", :all do |k,v|
+      all_values.concat(v)
+    end
+    assert_include all_values.flatten, "row1"
+    assert_include all_values.flatten, "a"
+    assert_include all_values.flatten, "aaa"
+
+    all_values = []
+    tsv.traverse "Id", :all do |k,v|
+      all_values.concat(v)
+    end
+    assert_include all_values.flatten, "row1"
+    assert_include all_values.flatten, "a"
+    assert_include all_values.flatten, "aaa"
+  end
+
+  def test_tsv_traverse_list
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:list
+#Id    ValueA    ValueB    OtherID
+row1    a    b    Id1
+row2    A    B    Id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB) do |k,v|
+      res[k] = v
+    end
+    assert_equal ["Id3", "B"], res["row2"]
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB), type: :double do |k,v|
+      res[k] = v
+    end
+    assert_equal [%w(Id3), %w(B)], res["row2"]
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB), type: :flat do |k,v|
+      res[k] = v
+    end
+    assert_equal %w(Id3 B), res["row2"]
+  end
+
+  def test_tsv_traverse_single
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:single
+#Id    ValueA
+row1    a
+row2    A
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    res = {}
+    tsv.traverse "ValueA", %w(Id) do |k,v|
+      res[k] = v
+    end
+    assert_equal "row1", res["a"]
+
+    res = {}
+    tsv.traverse "ValueA", %w(Id), type: :double do |k,v|
+      res[k] = v
+    end
+    assert_equal [["row1"]], res["a"]
+  end
+
+  def test_traverse_reorder_one2one
+    content =<<-'EOF'
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    res = {}
+    tsv.traverse "ValueA", one2one: true do |k,v|
+      res[k] = v
+    end
+
+    assert_include res["A2"][0], "row2"
+
+  end
+
+  def test_traverse_select
+    content =<<-'EOF'
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :persist => true, select: "B1")
+
+      assert_equal %w(row1), tsv.keys
+
+      tsv = TSV.open(filename, :persist => true, select: "B2")
+
+      assert_equal %w(row2), tsv.keys
+
+      tsv = TSV.open(filename, :persist => true, select: "B1")
+
+      assert_equal %w(row1), tsv.keys
+
+      tsv = TSV.open(filename, :persist => true, select: "B2")
+
+      assert_equal %w(row2), tsv.keys
+    end
   end
 end
 

data/test/scout/tsv/test_util.rb

@@ -0,0 +1,24 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVUtil < Test::Unit::TestCase
+  def test_open_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :sep => " " )
+    end
+    assert_equal %w(row1 row2), tsv.collect{|k,v| k }
+    assert NamedArray === tsv.collect{|k,v| v }.first
+    assert "row1", tsv["row1"].key
+  end
+
+end
+

data/test/scout/tsv/util/test_filter.rb

@@ -0,0 +1,188 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVFilters < Test::Unit::TestCase
+  def test_collect
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      assert_equal 2, tsv.collect.size
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+      assert_equal 1, tsv.collect.size
+    end
+  end
+
+  def test_through
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+    end
+  end
+
+  def test_2_filters
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+row3    A    C
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 2, elem.size
+      assert_equal ["row2", "row3"], elem.sort
+
+      tsv.add_filter "field:ValueB", ["C"]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+    end
+  end
+
+  def test_filter_persistence
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+row3    A    C
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"], tmpdir.fieldValueA.find(:user)
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 2, elem.size
+
+      tsv.add_filter "field:ValueB", ["C"]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"], tmpdir.fieldValueA.find(:user)
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 2, elem.size
+    end
+  end
+
+  def __test_filter_persistence_update
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"], Scout.tmp.test.Filter.fieldValueA.find(:user)
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+
+      tsv["row3"] = [["A"], ["C"]]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 2, elem.size
+
+      tsv.add_filter "field:ValueB", ["C"]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"], Scout.tmp.test.Filter.fieldValueA.find(:user)
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+    end
+  end
+
+  def test_delete
+    content =<<-EOF
+#ID ValueA ValueB Comment
+row1 a b c
+row2 A B C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), type: :double, :sep => /\s/)
+      assert_equal 2, tsv.keys.length
+      tsv.delete "row2"
+      assert_equal 1, tsv.keys.length
+
+      tsv = TSV.open(File.open(filename), type: :double, :sep => /\s/)
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+
+      assert_equal 1, tsv.keys.length
+      assert_equal ["row2"], tsv.keys
+
+      tsv.delete "row2"
+      assert_equal 0, tsv.keys.length
+
+      tsv.pop_filter
+      assert_equal ["row1"], tsv.keys
+    end
+  end
+
+  def test_filename
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+      assert tsv.filename =~ /ValueA/
+      tsv.pop_filter
+      assert tsv.filename !~ /ValueA/
+    end
+ 
+  end
+end
+

data/test/scout/tsv/util/test_process.rb

@@ -0,0 +1,47 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVProcess < Test::Unit::TestCase
+  def test_process
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    tsv.process "ValueA" do |v|
+      v.collect{|e| e.upcase }
+    end
+
+    assert_equal %w(A AA AAA), tsv["row1"][0]
+  end
+
+  def test_add_field
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    tsv.add_field "ValueC" do |k,v|
+      v[0].collect{|e| e.gsub("a", "c").gsub("A", "C") }
+    end
+
+    assert_equal %w(c cc ccc), tsv["row1"]["ValueC"]
+    assert_equal %w(C CC), tsv["row2"]["ValueC"]
+  end
+end
+