scout-gear 10.4.0 → 10.7.0

data/test/scout/tsv/test_attach.rb

@@ -31,6 +31,31 @@ row3    B    Id3
     end
   end
 
+  def test_attach_single
+    content1 =<<-EOF
+#: :sep=","
+#ID,ValueA
+row1,a
+row2,A
+    EOF
+
+    content2 =<<-EOF
+#: :sep=","
+#ID,ValueB
+row1,b
+row3,B
+    EOF
+
+    TmpFile.with_file(content1) do |filename1|
+      TmpFile.with_file(content2) do |filename2|
+        tsv = TSV.open(filename1, type: :single)
+        other = TSV.open(filename2, type: :single)
+        tsv = tsv.attach other, :complete => true
+        assert_equal 'b', tsv["row1"]["ValueB"]
+      end
+    end
+  end
+
   def test_attach_by_key
     content1 =<<-EOF
 #: :sep=" "
@@ -162,7 +187,7 @@ B    Id3
       tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
     end
 
-    tsv1.attach tsv2, bar: true
+    tsv1 = tsv1.attach tsv2, bar: true
 
     assert_equal %w(ValueA ValueB OtherID), tsv1.fields
     assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
@@ -224,4 +249,233 @@ row3        Id3
       end
     end
   end
+
+  def test_attach_index
+    content1 =<<-EOF
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#ValueE    OtherID
+e    Id1|Id2
+E    Id3
+    EOF
+
+    content_index =<<-EOF
+#Id    ValueE
+row1    e
+row2    E
+    EOF
+
+    tsv1 = tsv2 = index = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
+    end
+
+    TmpFile.with_file(content_index) do |filename|
+      index = TSV.open(File.open(filename), type: :flat, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, index: index
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
+
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
+    end
+
+    tsv1.attach tsv2, index: index
+
+    assert_equal %w(ValueA ValueB OtherID), tsv1.fields
+    assert_equal "Id1", tsv1["row1"]["OtherID"]
+  end
+
+  def test_attach_complete_identifiers
+    content1 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueA
+row1    a|aa|aaa
+row2    A
+    EOF
+
+    content2 =<<-EOF
+#: :sep=/\\s+/
+#Id2    ValueB
+ROW_1    b
+ROW_2    C
+    EOF
+
+    identifiers =<<-EOF
+#: :sep=/\\s+/
+#Id    Id2
+row1    ROW_1
+row2    ROW_2
+row3    ROW_3
+    EOF
+    Scout.claim Scout.tmp.test_tmpdir.test1.data, :string, content1
+    Scout.claim Scout.tmp.test_tmpdir.test2.data, :string, content2
+    Scout.claim Scout.tmp.test_tmpdir.identifiers.data, :string, identifiers
+
+    tsv1 = tsv2 = nil
+
+    tsv1 = Scout.tmp.test_tmpdir.test1.data.produce(true).tsv type: :double,  :sep => /\s+/
+    tsv2 = Scout.tmp.test_tmpdir.test2.data.produce(true).tsv type: :double,  :sep => /\s+/
+    ids = Scout.tmp.test_tmpdir.identifiers.data.produce(true).tsv type: :double,  :sep => /\s+/
+
+    tsv1.identifiers = ids
+
+    tsv1 = tsv1.attach tsv2
+
+    assert_equal [["A"], ["C"]], tsv1["row2"]
+
+    tsv1 = Scout.tmp.test_tmpdir.test1.data.produce(true).tsv type: :double,  :sep => /\s+/
+    tsv2 = Scout.tmp.test_tmpdir.test2.data.produce(true).tsv type: :double,  :sep => /\s+/
+    ids = Scout.tmp.test_tmpdir.identifiers.data.produce(true).tsv  type: :double,  :sep => /\s+/
+
+    tsv1.identifiers = ids
+
+    tsv1 = tsv1.attach tsv2, :complete => true
+    assert_equal [["A"], ["C"]], tsv1["row2"]
+  end
+
+  def test_attach_index_both_non_key
+    content1 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#: :sep=/\\s+/
+#ValueE    OtherID
+e    Id1|Id2
+E    Id3
+    EOF
+
+    content_index =<<-EOF
+#: :sep=/\\s+/
+#ValueA    OtherID
+a    Id1
+A    Id3
+    EOF
+
+    tmpdir = Scout.tmp.test_tmp
+
+    Scout.claim tmpdir.test1.data, :string, content1
+    Scout.claim tmpdir.test2.data, :string, content2
+    Scout.claim tmpdir.test2.identifiers, :string, content_index
+
+    tsv1 = tsv2 = nil
+
+    tsv1 = tmpdir.test1.data.produce(true).tsv type: :double,  :sep => /\s+/
+    tsv2 = tmpdir.test2.data.produce(true).tsv type: :double,  :sep => /\s+/
+
+    tsv2.identifiers = tmpdir.test2.identifiers.produce(true).produce.find #.to_s
+
+    tsv1.attach tsv2, :fields => ["ValueE"] #, :persist_input => true
+    assert_equal [["a", "aa", "aaa"], ["b"], ["e"]], tsv1["row1"]
+  end
+
+  def test_attach_single_nils
+    content1 =<<-EOF
+#Id,ValueA
+row1,
+row2,AA
+    EOF
+    content2 =<<-EOF
+#Id,ValueB
+row1,B
+row2,BB
+    EOF
+    content3 =<<-EOF
+#Id,ValueC
+row1,
+row2,CC
+    EOF
+
+    tsv1 = tsv2 = tsv3 = tsv4 = index = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), :sep => ',', :type => :single)
+      tsv1.keys.each{|k| tsv1[k] = nil if tsv1[k] == ""}
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), :sep => ',', :type => :single)
+      tsv2.keys.each{|k| tsv2[k] = nil if tsv2[k] == ""}
+    end
+
+    TmpFile.with_file(content3) do |filename|
+      tsv3 = TSV.open(File.open(filename), :sep => ',', :type => :single)
+      tsv3.keys.each{|k| tsv3[k] = nil if tsv3[k] == ""}
+    end
+
+    tmp = tsv1.attach(tsv2, :complete => true)
+    tmp = tmp.attach(tsv3, :complete => true)
+    assert_equal [nil, "B", nil], tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"]
+    assert_equal [nil, "B", nil], tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"]
+  end
+
+  def test_attach_flat
+    content1 =<<-EOF
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#ValueA    OtherID
+a    Id1|Id2
+A    Id3
+    EOF
+
+    tsv1 = tsv2 = index = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :flat, fields: ["ValueA"], sep: /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :double, sep: /\s+/)
+    end
+
+    res = tsv1.attach tsv2, :fields => ["OtherID"]
+    assert res["row2"].include? "Id3"
+    assert ! res["row2"].include?("b")
+  end
+
+  def test_attach_list_to_list_with_complete
+    content1 =<<-EOF
+#Id    ValueA    ValueB
+row1    a    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#Id    ValueC
+row1    c
+row2    C
+row3    CC
+    EOF
+
+    tsv1 = tsv2 = index = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, fields: ["ValueA"], sep: /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :list, sep: /\s+/)
+    end
+
+    res = tsv1.attach tsv2, :fields => ["ValueC"], complete: true
+    assert res["row2"].include?("C")
+    refute res["row2"].include?("b")
+    assert res["row3"].include?("CC")
+  end
 end

data/test/scout/tsv/test_change_id.rb

@@ -94,5 +94,30 @@ row2    C2|C22    D2|D22
     assert_equal ["C1","C11"], res["row1"]["ValueC"]
     assert_equal ["C2","C22"], res["row2"]["ValueC"]
   end
+
+  def test_translate
+
+    f1=<<-EOF
+#: :sep=' '
+#A B C
+a b c
+aa bb cc
+    EOF
+
+    identifiers=<<-EOF
+#: :sep=' '
+#A X
+a x
+aa xx
+    EOF
+
+    TmpFile.with_file(f1) do |tf1|
+      TmpFile.with_file(identifiers) do |ti|
+        tsv = TSV.open tf1, :identifiers => ti
+
+        assert TSV.change_key(tsv, "X").include? "x"
+      end
+    end
+  end
 end
 

data/test/scout/tsv/test_csv.rb

@@ -0,0 +1,50 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+require 'scout/tsv'
+
+class TestTSVCSV < Test::Unit::TestCase
+  def test_csv
+    text =<<-EOF
+Key,FieldA,FieldB
+k1,a,b
+k2,aa,bb
+    EOF
+
+    tsv = TSV.csv(text)
+    assert_equal 'bb', tsv['k2']['FieldB']
+  end
+
+  def test_csv_key_field
+    text =<<-EOF
+Key,FieldA,FieldB
+k1,a,b
+k2,aa,bb
+    EOF
+
+    tsv = TSV.csv(text, :key_field => 'FieldA', :type => :list)
+    assert_equal 'bb', tsv['aa']['FieldB']
+  end
+
+  def test_csv_double
+    text =<<-EOF
+Key,FieldA,FieldB
+k1,a,b
+k2,aa,bb
+    EOF
+
+    tsv = TSV.csv(text, :key_field => 'FieldA', :type => :double)
+    assert_equal ['bb'], tsv['aa']['FieldB']
+  end
+
+  def test_csv_noheader
+    text =<<-EOF
+k1,a,b
+k2,aa,bb
+    EOF
+
+    tsv = TSV.csv(text, :headers => false)
+    assert_equal %w(k2 aa bb), tsv['row-1']
+  end
+
+end
+

data/test/scout/tsv/test_dumper.rb

@@ -17,6 +17,22 @@ a\t1|11\t2|22
     assert_equal txt, dumper.stream.read
   end
 
+  def test_set_stream
+    io = StringIO.new
+    dumper = TSV::Dumper.new :key_field => "Key", :fields => %w(Field1 Field2), :type => :double
+    dumper.set_stream io
+    dumper.init
+    dumper.add "a", [["1", "11"], ["2", "22"]]
+    txt=<<-EOF
+#: :type=:double
+#Key\tField1\tField2
+a\t1|11\t2|22
+    EOF
+    dumper.close
+    io.rewind
+    assert_equal txt, io.read
+  end
+
   def test_to_s
     tsv = TSV.setup({}, :key_field => "Key", :fields => %w(Field1 Field2), :type => :double)
     tsv["a"] = [["1", "11"], ["2", "22"]]
@@ -40,5 +56,27 @@ a\t1|11\t2|22
       TSV.open(dumper.stream, bar: true)
     end
   end
+
+  def test_to_s_sort
+    tsv = TSV.setup({}, :key_field => "Key", :fields => %w(Field1 Field2), :type => :double)
+    tsv["b"] = [["2", "22"], ["3", "33"]]
+    tsv["a"] = [["1", "11"], ["2", "22"]]
+    txt=<<-EOF
+#: :type=:double
+#Key\tField1\tField2
+a\t1|11\t2|22
+b\t2|22\t3|33
+    EOF
+    assert_equal txt, tsv.to_s(keys: tsv.keys.sort)
+  end
+
+  def test_filename
+    tsv = datadir_test.person.marriages.tsv
+    assert tsv.filename
+
+    tsv2 = TSV.open(tsv.dumper_stream)
+    assert tsv2.filename
+  end
+
 end
 

data/test/scout/tsv/test_index.rb

@@ -198,5 +198,87 @@ row2    a    b    id3
       end
     end
   end
+
+  def test_index_fields
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :key_field => "OtherID", :persist => false)
+      index = tsv.index(:persist => true, :persist_update => true)
+      assert index["row1"].include? "Id1"
+      assert_equal "OtherID", index.fields.first
+    end
+  end
+
+
+  def test_simple_index_key_field
+    text=<<-EOF
+#: :sep=' '
+#Y X
+y x
+yy xx
+    EOF
+
+    TmpFile.with_file(text) do |tmp|
+      assert_equal "Y", TSV.open(tmp).index(:target => "X", :fields => ["Y"]).key_field
+      assert_equal "Y", TSV.index(tmp, :target => "X", :fields => ["Y"]).key_field
+    end
+  end
+
+  def test_pos_and_range_index
+    content =<<-EOF
+#Id	ValueA    ValueB    Pos1    Pos2
+row1    a|aa|aaa    b    0|10    10|30
+row2    A    B    30   35
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), type: :double, sep: /\s+/)
+      index = tsv.pos_index("Pos1")
+      assert_equal ["row1"], index[10]
+
+      index = tsv.range_index("Pos1", "Pos2")
+      assert_equal ["row1"], index[20]
+    end
+  end
+
+
+  def test_index_static_persist
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b|A    Id1
+row2    A    a|B    Id3
+row3    A    a|B    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "OtherID", :sep => /\s+/, :order => true, :persist => false)
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
+      assert_equal "OtherID", index.fields.first
+
+      index = TSV.index(filename, :target => "OtherID", :sep => /\s+/, :order => true, :persist => true)
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
+      assert_equal "OtherID", index.fields.first
+
+      Open.write(filename, Open.read(filename).sub(/row1.*Id1\n/,''))
+
+      index = TSV.index(filename, :target => "OtherID", :sep => /\s+/, :order => true, :persist => true)
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
+      assert_equal "OtherID", index.fields.first
+      assert index.include?('aaa')
+
+      index = TSV.index(filename, :target => "OtherID", :sep => /\s+/, :order => true, :persist => false)
+      assert ! index.include?('aaa')
+    end
+  end
+
+
 end
 

data/test/scout/tsv/test_open.rb

@@ -163,5 +163,49 @@ row2 aa bb cc
 
     assert_equal size, step.run.length
   end
+
+  def test_traverse_priority
+    require 'fc'
+
+    queue = FastContainers::PriorityQueue.new(:min)
+
+    array = []
+    100.times do e = rand(1000).to_i; array << e; queue.push(e,e) end
+
+    res = Open.traverse queue, :into => [] do |v|
+      v
+    end
+
+    assert_equal array.sort, res
+  end
+
+  def test_traverse_into_path
+    size = 100
+    array = (1..size).to_a.collect{|n| n.to_s}
+    TmpFile.with_file do |tmpfile|
+      Path.setup(tmpfile)
+      io = TSV.traverse array, :into => tmpfile do |e|
+        e
+      end
+      io.join
+      assert_equal size, Open.read(tmpfile).split("\n").length
+    end
+  end
+
+  def test_simple_index_key_field
+    text=<<-EOF
+#: :sep=' '
+#Y X
+y x
+yy xx
+    EOF
+
+    TmpFile.with_file(text) do |tmp|
+      k, f = Open.traverse TSV.open(tmp), key_field: "Y", fields: ["X"] do end
+      assert_equal "Y", k
+      k, f = Open.traverse tmp, key_field: "Y", fields: ["X"] do end
+      assert_equal "Y", k
+    end
+  end
 end
 

data/test/scout/tsv/test_parser.rb

@@ -115,6 +115,21 @@ k4 a|A b|B
     assert_equal 3, tsv.keys.length
   end
 
+  def test_parse_tsv_grep
+    content =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+k1 a|A b|B
+k2 a|A b|B
+k3 a|A b|B
+k4 a|A b|B
+    EOF
+    content = StringIO.new content
+
+    tsv = TSV.parse(content, :tsv_grep => ["k3","k4"])
+    assert_equal %w(k3 k4), tsv.keys.sort
+  end
 
   def test_parse_fields
     content =<<-EOF
@@ -247,4 +262,59 @@ k 1 2
       assert_equal [1.0, 2.0], data["k"]
     end
   end
+
+  def test_merge
+    content =<<-EOF
+#: :type=:double
+#PMID:Sentence number:TF:TG	Transcription Factor (Associated Gene Name)	Target Gene (Associated Gene Name)	Sign	Negation	PMID
+24265317:3:NR1H3:FASN	NR1H3	FASN			24265317
+17522048:0:NR1H3:FASN	NR1H3	FASN	+		17522048
+19903962:0:NR1H3:FASN	NR1H3	FASN			19903962
+19903962:7:NR1H3:FASN	NR1H3	FASN			19903962
+22183856:4:NR1H3:FASN	NR1H3	FASN			22183856
+22641099:4:NR1H3:FASN	NR1H3	FASN	+		22641099
+23499676:8:NR1H3:FASN	NR1H3	FASN	+		23499676
+11790787:5:NR1H3:FASN	NR1H3	FASN			11790787
+11790787:7:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:9:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:11:NR1H3:FASN	NR1H3	FASN			11790787
+17522048:1:NR1H3:FASN	NR1H3	FASN	+		17522048
+17522048:3:NR1H3:FASN	NR1H3	FASN			17522048
+22160584:1:NR1H3:FASN	NR1H3	FASN			22160584
+22160584:5:NR1H3:FASN	NR1H3	FASN	+		22160584
+22160584:8:NR1H3:FASN	NR1H3	FASN	+		22160584
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, key_field: "Transcription Factor (Associated Gene Name)", fields: ["Target Gene (Associated Gene Name)", "Sign", "PMID"], merge: true, one2one: true, type: :double)
+      assert_equal 16, tsv["NR1H3"]["Sign"].length
+    end
+  end
+
+  def test_load_stream
+    content =<<-EOF
+#Key    ValueA    ValueB
+k    a|A    b|B
+k1    a|A    b|B
+k2    a|A    b|B
+k3    a|A    b|B
+k4    a|A    b|B
+    EOF
+    content = StringIO.new content.gsub('    ', "\t")
+
+    TmpFile.with_file do |tmp_logfile|
+      old_logfile = Log.logfile
+      Log.logfile(tmp_logfile)
+      TmpFile.with_file do |persistence|
+        data = ScoutCabinet.open persistence, true
+        tsv = Log.with_severity(0) do
+          TSV.parse(content, data: data)
+        end
+        assert_equal %w(b B), tsv["k"]["ValueB"]
+        assert_equal %w(a A), tsv["k4"]["ValueA"]
+      end
+      Log.logfile(old_logfile)
+      assert Open.read(tmp_logfile).include?("directly into")
+    end
+  end
 end

data/test/scout/tsv/test_stream.rb

@@ -196,5 +196,27 @@ row2\tAAA
     assert_equal ["AA", "AAA"], tsv["row2"][0]
   end
 
+  def test_concat_streams
+
+    text1=<<-EOF
+#Key\tValueA
+row1\tA
+row2\tAA
+    EOF
+
+    text2=<<-EOF
+#Key\tValueA
+row3\tAAA
+row2\tBB
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.concat_streams([s1,s2]), :merge => true
+    assert_equal ["A"], tsv["row1"][0]
+    assert_equal ["AA","BB"], tsv["row2"][0]
+    assert_equal ["AAA"], tsv["row3"][0]
+  end
+
 end
 

data/test/scout/tsv/test_transformer.rb

@@ -3,6 +3,32 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/tsv'
 class TestTSVTransformer < Test::Unit::TestCase
+  def test_filename
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    TmpFile.with_file(content) do |file|
+      parser = TSV::Parser.new file
+      dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Values"], type: :flat, :filename => parser.options[:filename]
+      dumper.init
+
+      trans = TSV::Transformer.new parser, dumper
+      dumper = trans.traverse do |k,values|
+        [k, values.flatten]
+      end
+
+      tsv = trans.tsv
+      assert_include tsv.filename, File.dirname(file)
+
+      assert_equal %w(A1 A11 B1 B11), tsv['row1']
+    end
+    
+  end
+
   def test_traverse
     content =<<-EOF
 #: :sep=" "
@@ -40,12 +66,11 @@ row2    A2|A22    B2|B22
       dumper.init
 
       trans = TSV::Transformer.new parser, dumper
+      trans["row3"] = %w(A3 A33)
       dumper = trans.each do |k,values|
         values.replace values.flatten
       end
 
-      trans["row3"] = %w(A3 A33)
-
       tsv = trans.tsv
       assert_equal %w(A1 A11 B1 B11), tsv['row1']
       assert_equal %w(A3 A33), tsv['row3']
@@ -67,7 +92,6 @@ row2    A2|A22    B2|B22
       trans.type = :flat
       trans.sep = "\t"
 
-
       trans["row3"] = %w(A3 A33)
 
       trans.each do |k,values|