rbbt-util 3.2.1 → 4.0.0

data/test/rbbt/{util/tsv/test_filters.rb → tsv/test_filter.rb}

@@ -1,5 +1,5 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/util/tsv/filters'
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/tsv/filter'
 
 class TestTSVFilters < Test::Unit::TestCase
   def test_collect
@@ -11,7 +11,7 @@ row2    A    B
     EOF
 
     TmpFile.with_file(content1) do |filename|
-      tsv = TSV.new filename
+      tsv = TSV.open filename
       tsv.filter
       tsv.add_filter "field:ValueA", ["A"]
       assert_equal 1, tsv.collect.size
@@ -27,7 +27,7 @@ row2    A    B
     EOF
 
     TmpFile.with_file(content1) do |filename|
-      tsv = TSV.new filename
+      tsv = TSV.open filename
       tsv.filter
       tsv.add_filter "field:ValueA", ["A"]
       elem = []
@@ -46,7 +46,7 @@ row3    A    C
     EOF
 
     TmpFile.with_file(content1) do |filename|
-      tsv = TSV.new filename
+      tsv = TSV.open filename
       tsv.filter
       tsv.add_filter "field:ValueA", ["A"]
 
@@ -72,9 +72,9 @@ row3    A    C
     EOF
 
     TmpFile.with_file(content1) do |filename|
-      tsv = TSV.new filename
+      tsv = TSV.open filename
       tsv.filter
-      tsv.add_filter "field:ValueA", ["A"], Rbbt.tmp.test.Filter.fieldValueA
+      tsv.add_filter "field:ValueA", ["A"], Rbbt.tmp.test.Filter.fieldValueA.find(:user)
 
       elem = []
       tsv.through do |k,v| elem << k end
@@ -86,9 +86,9 @@ row3    A    C
       tsv.through do |k,v| elem << k end
       assert_equal 1, elem.size
 
-      tsv = TSV.new filename
+      tsv = TSV.open filename
       tsv.filter
-      tsv.add_filter "field:ValueA", ["A"], Rbbt.tmp.test.Filter.fieldValueA
+      tsv.add_filter "field:ValueA", ["A"], Rbbt.tmp.test.Filter.fieldValueA.find(:user)
 
       elem = []
       tsv.through do |k,v| elem << k end
@@ -105,7 +105,7 @@ row2    A    B
     EOF
 
     TmpFile.with_file(content1) do |filename|
-      tsv = TSV.new filename
+      tsv = TSV.open filename
       tsv.filter
       tsv.add_filter "field:ValueA", ["A"], Rbbt.tmp.test.Filter.fieldValueA.find(:user)
 
@@ -125,7 +125,7 @@ row2    A    B
       tsv.through do |k,v| elem << k end
       assert_equal 1, elem.size
 
-      tsv = TSV.new filename
+      tsv = TSV.open filename
       tsv.filter
       tsv.add_filter "field:ValueA", ["A"], Rbbt.tmp.test.Filter.fieldValueA.find(:user)
 
@@ -143,12 +143,12 @@ row2 A B C
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :double, :sep => /\s/)
+      tsv = TSV.open(File.open(filename), :double, :sep => /\s/)
       assert_equal 2, tsv.keys.length
       tsv.delete "row2"
       assert_equal 1, tsv.keys.length
 
-      tsv = TSV.new(File.open(filename), :double, :sep => /\s/)
+      tsv = TSV.open(File.open(filename), :double, :sep => /\s/)
       tsv.filter
       tsv.add_filter "field:ValueA", ["A"]
 
@@ -163,5 +163,23 @@ row2 A B C
     end
   end
 
+  def test_filename
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+      assert tsv.filename =~ /ValueA/
+      tsv.pop_filter
+      assert tsv.filename !~ /ValueA/
+    end
+ 
+  end
 end
 

data/test/rbbt/tsv/test_index.rb

@@ -0,0 +1,284 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/tsv'
+require 'rbbt/tsv/index'
+
+class TestTSVManipulate < Test::Unit::TestCase
+
+  def load_segment_data(data)
+    tsv = TSV.open(data, :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
+    tsv.add_field "Start" do |key, values|
+      values["Range"].first
+    end
+    tsv.add_field "End" do |key, values|
+      values["Range"].last
+    end
+
+    tsv = tsv.slice ["Start", "End"]
+ 
+    tsv
+  end
+
+  def test_index
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :key_field => "OtherID", :persistence => false)
+      index = tsv.index(:case_insensitive => true, :persistence => true)
+      assert index["row1"].include? "Id1"
+      assert_equal "OtherID", index.fields.first
+    end
+
+  end
+
+  def test_best_index
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b|A    Id1
+row2    A    a|B    Id3
+row3    A    a|B    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :key_field => "OtherID")
+      index = tsv.index(:order => true)
+      assert_equal "Id1", index['a'].first
+      assert_equal "Id3", index['A'].first
+      assert_equal "OtherID", index.fields.first
+      
+      index = tsv.index(:order => false)
+      assert_equal "Id1", index['a'].first
+    end
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :key_field => "OtherID")
+      index = tsv.index
+      assert index["row1"].include? "Id1"
+      assert_equal "OtherID", index.fields.first
+    end
+  end
+
+
+  def test_index_from_persit
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b|A    Id1
+row2    A    a|B    Id3
+row3    A    a|B    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(Open.open(filename), :sep => /\s+/, :key_field => "OtherID", :persist => true)
+      index = tsv.index(:order => true)
+      assert_equal "Id1", index['a'].first
+      assert_equal "Id3", index['A'].first
+      assert_equal "OtherID", index.fields.first
+    end
+  end
+
+  def test_index_to_persist
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b|A    Id1
+row2    A    a|B    Id3
+row3    A    a|B    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(Open.open(filename), :sep => /\s+/, :key_field => "OtherID", :persist => true)
+
+      index = tsv.index(:order => true, :persist => true)
+      assert_equal "Id1", index['a'].first
+      assert_equal "Id3", index['A'].first
+      assert_equal "OtherID", index.fields.first
+
+      tsv.delete "Id1"
+
+      index = tsv.index(:order => true, :persist => true)
+      assert_equal "Id1", index['a'].first
+      assert_equal "Id3", index['A'].first
+      assert_equal "OtherID", index.fields.first
+
+      index = tsv.index(:order => true, :persist => false)
+      assert_equal "Id3", index['a'].first
+    end
+  end
+
+  def test_index_static
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b|A    Id1
+row2    A    a|B    Id3
+row3    A    a|B    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => false)
+      assert_equal "Id1", index['a'].first
+      assert_equal "Id3", index['A'].first
+      assert_equal "OtherID", index.fields.first
+    end
+  end
+
+  def test_index_static_persist
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b|A    Id1
+row2    A    a|B    Id3
+row3    A    a|B    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => false)
+      assert_equal "Id1", index['a'].first
+      assert_equal "Id3", index['A'].first
+      assert_equal "OtherID", index.fields.first
+
+      index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => true)
+      assert_equal "Id1", index['a'].first
+      assert_equal "Id3", index['A'].first
+      assert_equal "OtherID", index.fields.first
+
+      Open.write(filename, Open.read(filename).sub(/row1.*Id1\n/,''))
+
+      index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => true)
+      assert_equal "Id1", index['a'].first
+      assert_equal "Id3", index['A'].first
+      assert_equal "OtherID", index.fields.first
+      assert index.include?('aaa')
+
+      index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => false)
+      assert ! index.include?('aaa')
+    end
+  end
+
+  def test_pos_index
+    content =<<-EOF
+#Id	ValueA    ValueB    Pos
+row1    a|aa|aaa    b    0|10
+row2    A    B    30
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :double, :sep => /\s+/)
+      index = tsv.pos_index("Pos")
+      assert_equal ["row1"], index[10]
+    end
+  end
+
+  def test_range_index
+    content =<<-EOF
+#Id	ValueA    ValueB    Pos1    Pos2
+row1    a|aa|aaa    b    0|10    10|30
+row2    A    B    30   35
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :double, :sep => /\s+/)
+      index = tsv.pos_index("Pos1")
+      assert_equal ["row1"], index[10]
+
+      index = tsv.range_index("Pos1", "Pos2")
+      assert_equal ["row1"], index[20]
+    end
+  end
+
+  def test_range_index2
+    data =<<-EOF
+#ID:Range
+#:012345678901234567890
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_segment_data(datafile)
+      f   = tsv.range_index("Start", "End", :persist => true)
+
+      assert_equal %w(), f[0].sort
+      assert_equal %w(b), f[1].sort
+      assert_equal %w(), f[20].sort
+      assert_equal %w(), f[(20..100)].sort
+      assert_equal %w(a b d), f[3].sort
+      assert_equal %w(a b c d e), f[(3..4)].sort
+    end
+  end
+
+  def test_range_index_persistent
+    data =<<-EOF
+#ID:Range
+#:012345678901234567890
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      TmpFile.with_file(load_segment_data(datafile)) do |tsvfile|
+        f = TSV.range_index(tsvfile, "Start", "End", :persistence => true)
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[1].sort
+        assert_equal %w(), f[20].sort
+        assert_equal %w(), f[(20..100)].sort
+        assert_equal %w(a b d), f[3].sort
+        assert_equal %w(a b c d e), f[(3..4)].sort
+      end
+    end
+  end
+
+
+#  #{{{ Test Attach
+#  def test_index_headerless
+#    content =<<-EOF
+#row1    a|aa|aaa    b    Id1|Id2
+#row2    A    B    Id3
+#    EOF
+#
+#    TmpFile.with_file(content) do |filename|
+#      tsv = TSV.open(File.open(filename), :sep => /\s+/)
+#      index = tsv.index(:target => 2)
+#      assert index["row1"].include? "Id1"
+#    end
+#  end
+
+
+
+  def test_range_index_persistent_with_filter
+    data =<<-EOF
+#ID:Range
+#:012345678901234567890
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      TmpFile.with_file(load_segment_data(datafile)) do |tsvfile|
+        f = TSV.range_index(tsvfile, "Start", "End", :filters => [["field:Start", "3"]])
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(), f[1].sort
+        assert_equal %w(), f[20].sort
+        assert_equal %w(), f[(20..100)].sort
+        assert_equal %w(a), f[3].sort
+        assert_equal %w(a), f[(3..4)].sort
+      end
+    end
+  end
+
+end
+

data/test/rbbt/{util/tsv → tsv}/test_manipulate.rb

@@ -1,23 +1,28 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/util/tsv'
-require 'rbbt/util/tsv/manipulate'
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../', 'test_helper.rb')
+require 'rbbt/tsv'
+require 'rbbt/tsv/manipulate'
 
 class TestTSVManipulate < Test::Unit::TestCase
 
-  def test_indentify_fields
+  def test_through
     content =<<-EOF
-#ID ValueA ValueB Comment
-row1 a b c
-row2 A B C
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row3    a    C    Id4
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :double, :sep => /\s/)
-      assert_equal :key, tsv.identify_field("ID")
+      tsv = TSV.open(File.open(filename), :sep => /\s+/)
+
+      new_key, new_fields = tsv.through "ValueA" do |key, values|
+        assert(tsv.keys.include? values["Id"].first)
+      end
+
+      assert_equal "ValueA", new_key
     end
   end
 
-
   def test_reorder_simple
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
@@ -27,41 +32,19 @@ row3    a    C    Id4
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :sep => /\s+/)
+      tsv = TSV.open(File.open(filename), :sep => /\s+/)
 
       tsv1 = tsv.reorder("ValueA")
 
       assert_equal "ValueA", tsv1.key_field
-      assert_equal %w(Id ValueB OtherID), tsv1.fields
       assert_equal ["B"], tsv1["A"]["ValueB"]
       assert_equal ["b","C"], tsv1["a"]["ValueB"]
       assert_equal ["b"], tsv1["aa"]["ValueB"]
+      assert_equal %w(Id ValueB OtherID), tsv1.fields
 
     end
   end
 
-  def test_reorder_simple_headerless
-    content =<<-EOF
-row1    a|aa|aaa    b    Id1|Id2
-row2    A    B    Id3
-row3    a    C    Id4
-    EOF
-
-    TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :sep => /\s+/)
-
-      tsv1 = tsv.reorder 0 
-
-      assert_nil tsv1.key_field
-      assert_equal ["B"], tsv1["A"][1]
-      assert_equal ["b","C"], tsv1["a"][1]
-      assert_equal ["b"], tsv1["aa"][1]
-      assert_equal ["row1"], tsv1["aa"][0]
-      assert_equal ["row1","row3"], tsv1["a"][0]
-    end
-  end
-
-
   def test_reorder_remove_field
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
@@ -71,7 +54,7 @@ row3    a    C    Id4
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :sep => /\s+/)
+      tsv = TSV.open(File.open(filename), :sep => /\s+/)
 
       tsv1 = tsv.reorder("ValueA", ["ValueB", "Id"])
 
@@ -84,24 +67,6 @@ row3    a    C    Id4
     end
   end
 
-  def test_through
-    content =<<-EOF
-#Id    ValueA    ValueB    OtherID
-row1    a|aa|aaa    b    Id1|Id2
-row2    A    B    Id3
-row3    a    C    Id4
-    EOF
-
-    TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :sep => /\s+/)
-
-      tsv.through "ValueA" do |key, values|
-        assert(tsv.keys.include? values["Id"].first)
-      end
-    end
-  end
-
-
   def test_slice
     content =<<-EOF
 #ID ValueA ValueB Comment
@@ -110,28 +75,11 @@ row2 A B C
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :double, :sep => /\s/)
+      tsv = TSV.open(File.open(filename), :type => :double, :sep => /\s/)
       assert_equal [["a"],["c"]], tsv.reorder(:key, ["ValueA", "Comment"])["row1"]
     end
   end
 
-  def test_sort
-    content =<<-EOF
-#Id    ValueA    ValueB    OtherID    Pos
-row1    a|aa|aaa    b    Id1|Id2    2
-row2    A    B    Id3    1
-row3    A|AA|AAA|AAA    B    Id3    3
-    EOF
-
-    TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :sep => /\s+/)
-      assert_equal %w(row1 row2 row3), tsv.sort
-      assert_equal %w(row1 row2 row3), tsv.sort(:key)
-      assert_equal %w(row2 row1 row3), tsv.sort("Pos")
-      assert_equal %w(row3 row2 row1), tsv.sort("ValueA") do |values| values["ValueA"].length end
-    end
-  end
-
   def test_select
      content =<<-EOF
 #Id    ValueA    ValueB    OtherID
@@ -141,7 +89,7 @@ row3    a    C    Id4
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(filename + '#:sep=/\s+/')
+      tsv = TSV.open(filename, :sep => /\s+/)
       assert tsv.type == :double
       
       new = tsv.select %w(b Id4)
@@ -156,13 +104,12 @@ row3    a    C    Id4
       new = tsv.select "ValueB" => /b|Id4/
       assert_equal %w(row1).sort, new.keys
 
-      tsv = TSV.new(filename + '#:sep=/\s+/#:type=:flat')
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :flat)
       assert tsv.type != :double
       
       new = tsv.select %w(b Id4)
       assert_equal %w(row1 row3).sort, new.keys.sort
 
-
       new = tsv.select do |k,v| v["ValueA"].include? "A" end
       assert_equal %w(row2).sort, new.keys.sort
     end
@@ -177,7 +124,7 @@ row3    a    C    Id4
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(File.open(filename), :sep => /\s+/)
+      tsv = TSV.open(File.open(filename), :sep => /\s+/)
 
       tsv.process "ValueA" do |field_values,key,values|
         field_values.collect{|v| "Pref:#{v}"}
@@ -196,7 +143,7 @@ row3    a    C    Id4
     EOF
  
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(filename + '#:sep=/\s+/')
+      tsv = TSV.open(filename, :sep => /\s+/)
       tsv.add_field "Str length" do |k,v| 
         (v.flatten * " ").length 
       end
@@ -205,23 +152,6 @@ row3    a    C    Id4
     end
   end
 
-  def test_slice_namespace
-     content =<<-EOF
-#Id    LetterValue:ValueA    LetterValue:ValueB    OtherID
-row1    a|aa|aaa    b    Id1|Id2
-row2    A    B    Id3
-row3    a    C    Id4
-    EOF
- 
-    TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(filename + '#:sep=/\s+/')
-
-      tsv_ns = tsv.slice_namespace "LetterValue"
-      assert_equal %w(LetterValue:ValueA LetterValue:ValueB), tsv_ns.fields
-    end
- 
-  end
-
   def test_add_field_double_with_list_result
      content =<<-EOF
 #Id    LetterValue:ValueA    LetterValue:ValueB    OtherID
@@ -231,7 +161,7 @@ row3    a    C    Id4
     EOF
  
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(filename + '#:sep=/\s+/')
+      tsv = TSV.open(filename, :sep => /\s+/)
 
       tsv.add_field "Test" do
         "test"
@@ -239,26 +169,26 @@ row3    a    C    Id4
 
       assert Array === tsv["row1"]["Test"]
     end
- 
   end
 
-  def test_through_single
+  def test_through_headless
      content =<<-EOF
-#Id    LetterValue
-row1    a1
-row2    A1
-row3    a2
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row3    a    C    Id4
     EOF
  
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.new(filename + '#:sep=/\s+/#:type=:single')
+      tsv = TSV.open(filename, :sep => /\s+/)
+
+      test = false
+      tsv.through do
+        test = true
+      end
+      assert test
 
-      #TODO: WTF!!!
-      assert_equal "row3", tsv.reorder("LetterValue", :key)[97]
     end
  
   end
 
-
 end
-