scout-gear 6.0.0 → 7.2.0

data/test/scout/tsv/persist/test_tokyocabinet.rb

@@ -0,0 +1,92 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVTokyo < Test::Unit::TestCase
+  def test_tokyo
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      Persist.persist(__method__, :HDB) do
+        TSV.open(filename)
+      end
+    end
+
+    assert_equal %w(a aa aaa), tsv["row1"][0]
+
+    tsv_loaded = assert_nothing_raised do
+      TmpFile.with_file(content) do |filename|
+        Persist.persist(__method__, :HDB) do
+          raise
+        end
+      end
+    end
+
+    assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
+  end
+
+  def test_custom_load
+    tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+
+    size = 100_000
+    (0..size).each do |i|
+      k = "key-#{i}"
+      values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+      values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+      tsv[k] = [values1, values2]
+    end
+
+    tc = Persist.persist(__method__, :HDB) do |file|
+      tsv
+    end
+
+    100.times do
+      i = rand(size).floor
+      assert_equal tc["key-#{i}"], tsv["key-#{i}"]
+    end
+  end
+
+  def test_speed
+    tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+
+    size = 100_000
+    (0..size).each do |i|
+      k = "key-#{i}"
+      values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+      values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+      tsv[k] = [values1, values2]
+    end
+
+    tc = Persist.persist(__method__, :HDB) do |file|
+      data = ScoutCabinet.open(file, true, "HDB")
+      TSV.setup(data, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
+      data.extend TSVAdapter
+      Log::ProgressBar.with_bar size do |b|
+        (0..size).each do |i|
+          b.tick
+          k = "key-#{i}"
+          values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
+          values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
+
+          data[k] = [values1, values2]
+        end
+      end
+      data
+    end
+
+    100.times do
+      i = rand(size).floor
+      assert_equal tc["key-#{i}"], tsv["key-#{i}"]
+    end
+  end
+end
+

data/test/scout/tsv/test_dumper.rb

@@ -0,0 +1,44 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVDumper < Test::Unit::TestCase
+  def test_dumper
+    dumper = TSV::Dumper.new :key_field => "Key", :fields => %w(Field1 Field2), :type => :double
+    dumper.init
+    dumper.add "a", [["1", "11"], ["2", "22"]]
+    txt=<<-EOF
+#: :type=:double
+#Key\tField1\tField2
+a\t1|11\t2|22
+    EOF
+    dumper.close
+    assert_equal txt, dumper.stream.read
+  end
+
+  def test_to_s
+    tsv = TSV.setup({}, :key_field => "Key", :fields => %w(Field1 Field2), :type => :double)
+    tsv["a"] = [["1", "11"], ["2", "22"]]
+    txt=<<-EOF
+#: :type=:double
+#Key\tField1\tField2
+a\t1|11\t2|22
+    EOF
+    assert_equal txt, tsv.to_s
+  end
+
+  def test_raise
+    dumper = TSV::Dumper.new :key_field => "Key", :fields => %w(Field1 Field2), :type => :double
+    dumper.init
+    t = Thread.new do
+      dumper.add "a", [["1", "11"], ["2", "22"]]
+      dumper.abort ScoutException
+    end
+
+    assert_raise ScoutException do
+      dumper.stream.read
+    end
+  end
+end
+

data/test/scout/tsv/test_index.rb

@@ -0,0 +1,64 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestTSVIndex < Test::Unit::TestCase
+  def test_true
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3|a
+row2    a    b    id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "ValueB")
+      assert_equal 'b', index["row1"]
+      assert_equal 'b', index["a"]
+      assert_equal 'b', index["aaa"]
+      assert_equal 'B', index["A"]
+    end
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "ValueB", :fields => "OtherID")
+      assert_equal 'B', index["a"]
+      assert_nil index["B"]
+    end
+  end
+
+  def test_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3|a
+row2    a    b    id3
+    EOF
+    tsv = TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "ValueB", :persist => true)
+      assert_equal 'b', index["row1"]
+      assert_equal 'b', index["a"]
+      assert_equal 'b', index["aaa"]
+      assert_equal 'B', index["A"]
+    end
+  end
+
+  def __test_speed
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3|a
+row2    a    b    id3
+    EOF
+    tsv = TmpFile.with_file(content) do |filename|
+      Misc.benchmark 1000 do
+        TSV.index(filename, :target => "ValueB")
+      end
+      Misc.benchmark 1000 do
+        TSV.index(filename, :target => "ValueB", order: false)
+      end
+    end
+  end
+end
+

data/test/scout/tsv/test_parser.rb

@@ -0,0 +1,173 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVParser < Test::Unit::TestCase
+
+  def test_parse_line
+    line = (0..10).to_a * "\t"
+    key, values = TSV.parse_line(line)
+
+    assert_equal "0", key
+    assert_equal (1..10).collect{|v| v.to_s }, values
+  end
+
+  def test_parse_line_key
+    line = (0..10).to_a * "\t"
+    key, values = TSV.parse_line(line, key: 2)
+    
+    assert_equal "2", key
+    assert_equal %w(0 1 3 4 5 6 7 8 9 10), values
+  end
+
+
+  def test_parse_double
+    line = (0..10).collect{|v| v == 0 ? v : [v,v] * "|" } * "\t"
+    key, values = TSV.parse_line(line, type: :double, cast: :to_i)
+
+    assert_equal "0", key
+    assert_equal (1..10).collect{|v| [v,v] }, values
+  end
+
+  def __test_benchmark
+    num = 10_000
+    txt = num.times.inject(nil) do |acc,i|
+      (acc.nil? ? "" : acc << "\n") << (0..10).collect{|v| v == 0 ? i : [v,v] * "|" } * "\t"
+    end 
+
+    txt = StringIO.new(([txt] * (10))*"\n")
+    Misc.benchmark 1 do
+    #Misc.profile do
+      data = TSV.parse_stream(txt, fix: true, type: :double, bar: true, merge: :concat)
+      assert_equal num, data.size
+      assert_equal 20, data['1'][0].length
+    end
+  end
+
+  def test_parse_stream
+    lines =<<-EOF
+1 2 3 4 5
+11 12 13 14 15
+    EOF
+
+    lines = StringIO.new lines
+
+    data = TSV.parse_stream lines, sep: " "
+    assert_equal data["1"], %w(2 3 4 5)
+  end
+
+  def test_parse_stream_block
+    lines =<<-EOF
+1 2 3 4 5
+11 12 13 14 15
+    EOF
+
+    lines = StringIO.new lines
+
+    sum = 0
+    res = TSV.parse_stream(lines, sep: " ") do |k,values|
+      sum += values.inject(0){|acc,i| acc += i.to_i }
+    end
+    assert_equal 68, sum
+  end
+
+  def test_parse_header
+    header =<<-EOF
+#: :sep=" "
+#Key ValueA ValueB
+k A B
+    EOF
+    header = StringIO.new header
+
+    assert_equal "Key", TSV.parse_header(header)[1]
+  end
+
+  def test_parse
+    header =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+    EOF
+    header = StringIO.new header
+
+    tsv = TSV.parse(header)
+    assert_equal 'a', tsv['k'][0][0]
+  end
+
+  def test_parse_fields
+    content =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+    EOF
+    content = StringIO.new content
+
+    tsv = TSV.parse(content, fields: %w(ValueB))
+    assert_equal [%w(b B)], tsv['k']
+    assert_equal %w(ValueB), tsv.fields
+
+    content.rewind
+
+    tsv = TSV.parse(content, fields: %w(ValueB ValueA))
+    assert_equal [%w(b B), %w(a A)], tsv['k']
+    assert_equal %w(ValueB ValueA), tsv.fields
+
+    content.rewind
+
+    tsv = TSV.parse(content, fields: %w(ValueB Key))
+    assert_equal [%w(b B), %w(k)], tsv['k']
+  end
+
+  def test_parse_key
+    content =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+    EOF
+    content = StringIO.new content
+
+    tsv = TSV.parse(content, key_field: "ValueB")
+    assert_equal %w(b B), tsv.keys
+    assert_equal %w(a A), tsv["B"][1]
+
+    content.rewind
+
+    tsv = TSV.parse(content, key_field: "ValueB", one2one: true, type: :double)
+    assert_equal %w(b B), tsv.keys
+    assert_equal %w(A), tsv["B"][1]
+
+    content.rewind
+
+    tsv = TSV.parse(content, key_field: "ValueB", one2one: true, type: :list)
+    assert_equal %w(b B), tsv.keys
+    assert_equal "a", tsv["b"][1]
+    assert_equal "A", tsv["B"][1]
+    assert_equal "k", tsv["b"][0]
+    assert_equal "k", tsv["B"][0]
+
+    content.rewind
+
+    tsv = TSV.parse(content, key_field: "ValueB", one2one: true, type: :list)
+    assert_equal %w(b B), tsv.keys
+    assert_equal "A", tsv["B"][1]
+  end
+
+  def test_parser_class
+    content =<<-EOF
+Key ValueA ValueB
+k a|A b|B
+    EOF
+    content = StringIO.new content
+
+    parser = TSV::Parser.new content, sep: " ", header_hash: ''
+
+    assert_equal "Key", parser.key_field
+
+    values = []
+    parser.traverse fields: %w(ValueB), type: :double do |k,v|
+      values << [k,v]
+    end
+
+    assert_equal [["k", [%w(b B)]]], values
+  end
+end

data/test/scout/tsv/test_persist.rb

@@ -0,0 +1,36 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVPersist < Test::Unit::TestCase
+  def test_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      TmpFile.with_file(content) do |filename|
+        TSV.open(filename)
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+
+    assert_nothing_raised do
+      tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+        raise
+      end
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+end
+

data/test/scout/tsv/test_traverse.rb

@@ -0,0 +1,9 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_true
+    assert true
+  end
+end
+

data/test/scout/work_queue/test_worker.rb

@@ -43,7 +43,7 @@ class TestQueueWorker < Test::Unit::TestCase
             current = nil
             pid_list.each do |pid|
               if pid != current
-                raise "Out of order" if seen.include? pid
+                raise "Out of order #{Log.fingerprint seen} #{ pid }" if seen.include? pid
               end
               current = pid
               seen << pid
@@ -95,5 +95,53 @@ class TestQueueWorker < Test::Unit::TestCase
     output.clean
   end
 
+  def test_process_exception
+    input = WorkQueue::Socket.new
+    output = WorkQueue::Socket.new
+
+    workers = 10.times.collect{ WorkQueue::Worker.new }
+    workers.each do |w|
+      w.process(input, output) do |obj|
+        raise ScoutException
+        [Process.pid, obj.inspect] * " "
+      end
+    end
+
+    read = Thread.new do 
+      Thread.current.report_on_exception = false
+      begin
+        while obj = output.read
+          if DoneProcessing === obj
+            pid = obj.pid
+            @worker_mutex.synchronize{ @workers.delete_if{|w| w.pid = pid } }
+            break if workers.empty?
+          end
+          raise obj if Exception === obj
+        end
+      end
+    end
+
+    write = Thread.new do
+      Thread.report_on_exception = false
+      100.times do |i|
+        input.write i
+      end
+      10.times do
+        input.write DoneProcessing.new
+      end
+      input.close_write
+    end
+
+    write.join
+
+    assert_raise WorkerException do
+      read.join
+    end
+
+    WorkQueue::Worker.join workers
+    input.clean
+    output.clean
+  end
+
 end
 

data/test/scout/workflow/step/test_dependencies.rb

@@ -0,0 +1,25 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/named_array'
+class TestStepDependencies < Test::Unit::TestCase
+  def test_recursive_inputs
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, NamedArray.setup(["12"], %w(input1)) do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2, NamedArray.setup([2], %w(input2)) do |times|
+      step1 = dependencies.first
+      (step1.inputs.first + " has " + step1.load.to_s + " characters") * times
+    end
+
+    step2.dependencies = [step1]
+
+    assert_equal 2, step2.inputs["input2"]
+    assert_equal "12", step2.recursive_inputs["input1"]
+    assert_equal 2, step2.inputs[:input2]
+    assert_equal "12", step2.recursive_inputs[:input1]
+  end
+end
+

data/test/scout/workflow/step/test_info.rb

@@ -4,27 +4,25 @@ require 'scout/workflow'
 
 class TestStepInfo < Test::Unit::TestCase
   def test_dependency
-    sss 0 do
-      TmpFile.with_file do |tmpdir|
-        Path.setup(tmpdir)
-        tmpfile = tmpdir.test_step
-        step1 = Step.new tmpfile.step1, ["12"] do |s|
-          s.length
-        end
+    TmpFile.with_file do |tmpdir|
+      Path.setup(tmpdir)
+      tmpfile = tmpdir.test_step
+      step1 = Step.new tmpfile.step1, ["12"] do |s|
+        s.length
+      end
 
-        assert_equal 2, step1.exec
-        assert_equal 2, step1.run
+      assert_equal 2, step1.exec
+      assert_equal 2, step1.run
 
-        step2 = Step.new tmpfile.step2 do 
-          step1 = dependencies.first
-          step1.inputs.first + " has " + step1.load.to_s + " characters"
-        end
+      step2 = Step.new tmpfile.step2 do 
+        step1 = dependencies.first
+        step1.inputs.first + " has " + step1.load.to_s + " characters"
+      end
 
-        step2.dependencies = [step1]
+      step2.dependencies = [step1]
 
-        assert_equal "12 has 2 characters", step2.run
-        assert_equal "12 has 2 characters", step2.run
-      end
+      assert_equal "12 has 2 characters", step2.run
+      assert_equal "12 has 2 characters", step2.run
     end
   end
 end

data/test/scout/workflow/step/test_load.rb

@@ -5,29 +5,27 @@ require 'scout/workflow/step'
 
 class TestStepLoad < Test::Unit::TestCase
   def test_dependency
-    sss 0 do
-      tmpfile = tmpdir.test_step
-      step1 = Step.new tmpfile.step1, ["12"] do |s|
-        s.length
-      end
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
 
-      step2 = Step.new tmpfile.step2 do 
-        step1 = dependencies.first
-        step1.inputs.first + " has " + step1.load.to_s + " characters"
-      end
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
 
-      step2.dependencies = [step1]
+    step2.dependencies = [step1]
 
-      step2.recursive_clean
-      step2.run
+    step2.recursive_clean
+    step2.run
 
-      new_step2 = Step.load(step2.path)
+    new_step2 = Step.load(step2.path)
 
-      assert_equal "12 has 2 characters", new_step2.load
-      assert_equal "12 has 2 characters", new_step2.run
-      assert_equal 2, new_step2.dependencies.first.run
-      assert_equal "12", new_step2.dependencies.first.inputs.first
-    end
+    assert_equal "12 has 2 characters", new_step2.load
+    assert_equal "12 has 2 characters", new_step2.run
+    assert_equal 2, new_step2.dependencies.first.run
+    assert_equal "12", new_step2.dependencies.first.inputs.first
   end
 
   def test_relocate

data/test/scout/workflow/step/test_provenance.rb

@@ -0,0 +1,25 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+class TestStepProvenance < Test::Unit::TestCase
+  def test_true
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
+
+    step2.dependencies = [step1]
+
+    step2.run
+
+    assert_include Step.prov_report(step2), 'step1'
+    assert_include Step.prov_report(step2), 'step2'
+  end
+end
+