scout-gear 7.2.0 → 8.0.0

data/test/scout/tsv/test_attach.rb

@@ -0,0 +1,227 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVAttach < Test::Unit::TestCase
+  def test_attach_simple
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, :complete => true
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+        assert_equal %w(Id3), tsv["row3"]["OtherID"]
+        assert_equal %w(B), tsv["row3"]["ValueB"]
+      end
+    end
+  end
+
+  def test_attach_by_key
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    B1|B11    Id1|Id11
+row2.2    B2|B22|B222    Id2.2|Id22.2|Id222.2
+row3    B3    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, complete: true, match_key: "ValueB"
+        assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
+        assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
+        assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
+        assert_equal %w(Id2.2 Id22.2), tsv["row2"]["OtherID"]
+      end
+    end
+  end
+
+  def test_attach_by_reorder
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ValueB ID  OtherID
+B1  row1|row1.1  Id1|Id11
+B2 row2    Id2.2|Id22.2|Id222.2
+B3 row3   Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1)
+        other = TSV.open(filename2)
+        tsv.attach other, match_key: "ID", one2one: false
+        assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
+        assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
+        assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
+        assert_equal %w(Id2.2 Id22.2 Id222.2), tsv["row2"]["OtherID"]
+      end
+    end
+  end
+
+
+  def test_attach_same_key
+    content1 =<<-EOF
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, fields: "OtherID"
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
+
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, fields: "OtherID"
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal "Id1", tsv1["row1"]["OtherID"]
+  end
+
+  def test_attach_source_field
+    content1 =<<-EOF
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#ValueB    OtherID
+b    Id1|Id2
+B    Id3
+    EOF
+
+    tsv1 = tsv2 = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, bar: true
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
+
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal "Id1", tsv1["row1"]["OtherID"]
+  end
+
+  def test_attach_transformer
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#ID    ValueB    OtherID
+row1    b    Id1|Id2
+row3    B    Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        out = TSV.attach filename1, filename2, target: :stream, bar: false
+        tsv = out.tsv
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+      end
+    end
+  end
+
+  def test_attach_flexible_names
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=" "
+#Identifiers(ID) OtherID
+row1        Id1|Id2
+row3        Id3
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        out = TSV.attach filename1, filename2, target: :stream, bar: false
+        tsv = out.tsv
+        assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
+      end
+    end
+  end
+end

data/test/scout/tsv/test_change_id.rb

@@ -0,0 +1,98 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestChangeID < Test::Unit::TestCase
+  def test_simple_reorder
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TSV.open StringIO.new(content1)
+
+    res = tsv.change_key "ValueA", keep: true
+    assert_equal ["row1"], res["A1"]["ID"]
+    assert_equal ["row1"], res["A11"]["ID"]
+    assert_equal ["row2"], res["A2"]["ID"]
+
+    res = tsv.change_key "ValueA", keep: false, one2one: true
+    assert_equal ["B1"], res["A1"]["ValueB"]
+    assert_equal ["B11"], res["A11"]["ValueB"]
+  end
+
+  def test_simple_reorder_file
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    TmpFile.with_file(content1) do |file1|
+      res = TSV.change_key file1, "ValueA", keep: true
+      assert_equal ["row1"], res["A1"]["ID"]
+      assert_equal ["row1"], res["A11"]["ID"]
+      assert_equal ["row2"], res["A2"]["ID"]
+      assert_equal ["B1","B11"], res["A1"]["ValueB"]
+
+      res = TSV.change_key file1, "ValueA", one2one: true, keep: true
+      assert_equal ["row1"], res["A1"]["ID"]
+      assert_equal ["row1"], res["A11"]["ID"]
+      assert_equal ["row2"], res["A2"]["ID"]
+      assert_equal ["B1"], res["A1"]["ValueB"]
+    end
+  end
+
+  def test_change_key_identifiers
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    identifiers_content =<<-EOF
+#: :sep=" "
+#ID    ValueC    ValueD
+row1    C1|C11    D1|D11
+row2    C2|C22    D2|D22
+    EOF
+
+
+    tsv = TSV.open StringIO.new(content1)
+    identifiers = TSV.open StringIO.new(identifiers_content)
+
+    res = tsv.change_key "ValueC", identifiers: identifiers, keep: true
+    assert_equal ["row1"], res["C1"]["ID"]
+    assert_equal ["row1"], res["C11"]["ID"]
+    assert_equal ["row2"], res["C2"]["ID"]
+  end
+
+  def test_change_id_identifiers
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    identifiers_content =<<-EOF
+#: :sep=" "
+#ID    ValueC    ValueD
+row1    C1|C11    D1|D11
+row2    C2|C22    D2|D22
+    EOF
+
+
+    tsv = TSV.open StringIO.new(content1)
+    identifiers = TSV.open StringIO.new(identifiers_content)
+
+    res = tsv.change_id "ValueA", "ValueC", identifiers: identifiers
+    assert_equal ["C1","C11"], res["row1"]["ValueC"]
+    assert_equal ["C2","C22"], res["row2"]["ValueC"]
+  end
+end
+

data/test/scout/tsv/test_dumper.rb

@@ -37,7 +37,7 @@ a\t1|11\t2|22
     end
 
     assert_raise ScoutException do
-      dumper.stream.read
+      TSV.open(dumper.stream, bar: true)
     end
   end
 end

data/test/scout/tsv/test_index.rb

@@ -1,8 +1,25 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
+require 'scout/tsv'
 class TestTSVIndex < Test::Unit::TestCase
-  def test_true
+  def load_segment_data(data)
+    tsv = TSV.open(data, type: :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
+
+    tsv = tsv.add_field "Start" do |key, values|
+      values["Range"].first
+    end
+
+    tsv = tsv.add_field "End" do |key, values|
+      values["Range"].last
+    end
+
+    tsv = tsv.slice ["Start", "End"]
+ 
+    tsv
+  end
+
+  def test_index
     content =<<-'EOF'
 #: :sep=/\s+/#:type=:double#:merge=:concat
 #Id    ValueA    ValueB    OtherID
@@ -20,12 +37,45 @@ row2    a    b    id3
     end
 
     TmpFile.with_file(content) do |filename|
-      index = TSV.index(filename, :target => "ValueB", :fields => "OtherID")
+      index = TSV.index(filename, :target => "ValueB", :fields => ["OtherID"])
       assert_equal 'B', index["a"]
       assert_nil index["B"]
     end
   end
 
+  def test_from_tsv
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A|b    B    Id3|a
+row2    a    b    id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename)
+      index = TSV.index(tsv, :target => "ValueB")
+      assert_equal 'b', index["a"]
+      assert_equal 'B', index["B"]
+      assert_equal 'b', index["b"]
+
+      index = tsv.index(:target => "ValueB")
+      assert_equal 'b', index["a"]
+      assert_equal 'B', index["B"]
+      assert_equal 'b', index["b"]
+
+
+      index = TSV.index(tsv, :target => "ValueB", :fields => "OtherID")
+      assert_equal 'B', index["a"]
+      assert_nil index["B"]
+
+      index = tsv.index(:target => "ValueB", :fields => "OtherID")
+      assert_equal 'B', index["a"]
+      assert_nil  index["B"]
+    end
+  end
+
+
   def test_persist
     content =<<-'EOF'
 #: :sep=/\s+/#:type=:double#:merge=:concat
@@ -35,7 +85,7 @@ row2    A    B    Id3|a
 row2    a    b    id3
     EOF
     tsv = TmpFile.with_file(content) do |filename|
-      index = TSV.index(filename, :target => "ValueB", :persist => true)
+      index = TSV.index(filename, :target => "ValueB", :persist => true, bar: true)
       assert_equal 'b', index["row1"]
       assert_equal 'b', index["a"]
       assert_equal 'b', index["aaa"]
@@ -43,6 +93,80 @@ row2    a    b    id3
     end
   end
 
+  def test_range_index
+    data =<<-EOF
+# 012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_segment_data(datafile)
+      f   = tsv.range_index("Start", "End", :persist => true)
+
+      assert_equal %w(), f[0].sort
+      assert_equal %w(b), f[1].sort
+      assert_equal %w(), f[20].sort
+      assert_equal %w(), f[(20..100)].sort
+      assert_equal %w(a b d), f[3].sort
+      assert_equal %w(a b c d e), f[(3..4)].sort
+    end
+  end
+
+  def test_pos_index
+    data =<<-EOF
+# 012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_segment_data(datafile)
+      f   = tsv.pos_index("Start", :persist => true)
+
+      assert_equal %w(), f[0].sort
+      assert_equal %w(a c d e), f[(2..4)].sort
+    end
+  end
+
+  def test_range_index_persistent
+    data =<<-EOF
+# 012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      load_segment_data(datafile)
+      TmpFile.with_file(load_segment_data(datafile).to_s) do |tsvfile|
+        f = TSV.range_index(tsvfile, "Start", "End", :persist => true)
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[1].sort
+        assert_equal %w(), f[20].sort
+        assert_equal %w(), f[(20..100)].sort
+        assert_equal %w(a b d), f[3].sort
+        assert_equal %w(a b c d e), f[(3..4)].sort
+      end
+    end
+  end
+
+
   def __test_speed
     content =<<-'EOF'
 #: :sep=/\s+/#:type=:double#:merge=:concat

data/test/scout/tsv/test_open.rb

@@ -0,0 +1,167 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestOpenTraverse < Test::Unit::TestCase
+  def test_array
+    num_lines = 100
+    lines = num_lines.times.collect{|i| "line-#{i}" }
+
+    r = TSV.traverse lines, :into => [] do |l|
+      l + "-" + Process.pid.to_s
+    end
+
+    assert_equal num_lines, r.length
+  end
+
+  def test_array_cpus
+    num_lines = 1000
+    lines = num_lines.times.collect{|i| "line-#{i}" }
+
+    r = TSV.traverse lines, :into => [], :cpus => 2 do |l|
+      l + "-" + Process.pid.to_s
+    end
+
+    assert_equal num_lines, r.length
+    assert_equal 2, r.collect{|l| l.split("-").last}.uniq.length
+  end
+
+  def test_tsv_cpus
+    num_lines = 10000
+    lines = num_lines.times.collect{|i| "line-#{i}" }
+
+    tsv  = TSV.setup({}, key_field: "Line", :fields => %w(Prefix Number), :type => :list)
+    lines.each do |line|
+      tsv[line] = ["LINE", line.split("-").last]
+    end
+
+    r = TSV.traverse tsv, :into => [], :cpus => 2, :bar => {desc: "Process", severity: 0} do |l,v|
+      pre, num = v
+      pre + "-" + num.to_s + "-" + Process.pid.to_s
+    end
+
+    assert_equal num_lines, r.length
+    assert_equal 2, r.collect{|l| l.split("-").last}.uniq.length
+    assert_equal "LINE", r.collect{|l| l.split("-").first}.first
+  end
+
+  def test_into_stream
+    num_lines = 100
+    lines = num_lines.times.collect{|i| "line-#{i}" }
+
+    r = TSV.traverse lines, :into => :stream do |l|
+      l + "-" + Process.pid.to_s
+    end
+
+    assert_equal num_lines, r.read.split("\n").length
+  end
+
+  def test_into_stream_error
+    num_lines = 100
+    lines = num_lines.times.collect{|i| "line-#{i}" }
+
+    assert_raise ScoutException do
+      Log.with_severity 7 do
+        i = 0
+        r = TSV.traverse lines, :into => :stream, cpus: 3 do |l|
+          raise ScoutException if i > 10
+          i += 1
+          l + "-" + Process.pid.to_s
+        end
+
+        r.read
+      end
+    end
+  end
+
+  def test_into_dumper_error
+    num_lines = 100
+    lines = num_lines.times.collect{|i| "line-#{i}" }
+
+    assert_raise ScoutException do 
+      i = 0
+      Log.with_severity 7 do
+        dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Value"], :type => :single
+        dumper.init
+        dumper = TSV.traverse lines, :into => dumper, :cpus => 3 do |l|
+          raise ScoutException if i > 10
+          i += 1
+          value = l + "-" + Process.pid.to_s
+
+          [i.to_s, value]
+        end
+        ppp dumper.stream.read
+      end
+    end
+  end
+
+  def test_traverse_line
+    text=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row1 A B C
+row1 a b c
+row2 AA BB CC
+row2 aa bb cc
+    EOF
+
+    TmpFile.with_file(text) do |file|
+      lines = Open.traverse file, :type => :line, :into => [] do |line|
+        line
+      end
+      assert_include lines, "row2 AA BB CC"
+    end
+  end
+
+  def test_collapse_stream
+    text=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row1 A B C
+row1 a b c
+row2 AA BB CC
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    collapsed = TSV.collapse_stream(s)
+    tsv = TSV.open collapsed 
+    assert_equal ["A", "a"], tsv["row1"][0]
+    assert_equal ["BB", "bb"], tsv["row2"][1]
+  end
+
+  def test_cpus_error_dumper
+    num_lines = 100
+    lines = num_lines.times.collect{|i| "line-#{i}" }
+
+    dumper =  TSV::Dumper.new :key_field => "Key", :fields => ["Field"], type: :single
+    dumper.init
+    assert_raise ScoutException do
+      Log.with_severity 0 do
+        i = 0
+        TSV.traverse lines, :into => dumper, cpus: 3 do |l|
+          raise ScoutException if i > 10
+          i += 1
+          [Process.pid.to_s, l + "-" + Process.pid.to_s]
+        end
+
+      end
+      ppp dumper.stream.read
+    end
+  end
+
+  def test_step_travese_cpus
+
+    size = 1000
+    step = Step.new tmpdir.step[__method__] do
+      lines = size.times.collect{|i| "line-#{i}" }
+      Open.traverse lines, :type => :array, :into => :stream, :cpus => 3 do |line|
+        line.reverse
+      end
+    end
+    step.type = :array
+
+    assert_equal size, step.run.length
+  end
+end
+