scout-gear 10.4.0 → 10.6.1

data/test/scout/tsv/test_traverse.rb

@@ -197,5 +197,83 @@ row2    A2|A22    B2|B22
       assert_equal %w(row2), tsv.keys
     end
   end
+
+  def test_traverse_named
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row3    a    C    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/)
+
+      new_key, new_fields = tsv.through :key, ["ValueB"] do |key, values|
+        assert_include %w(b B C), values["ValueB"].first
+      end
+    end
+  end
+
+  def test_traverse_change_key
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row3    a    C    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/)
+
+      new_key, new_fields = tsv.through "ValueA" do |key, values|
+        assert_include tsv.keys, values["Id"].first
+      end
+
+      assert_equal "ValueA", new_key
+    end
+  end
+
+  def test_traverse_flat
+    content =<<-EOF
+#: :type=:flat
+#Row   vA
+row1    a    b    Id1
+row2    A    B    Id3
+row3    a    C    Id4
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :flat)
+      keys = []
+      tsv.through "vA" do |k,v|
+        keys << k
+      end
+      assert_include keys, "B"
+
+      tsv.through :key do |k,v|
+        assert_equal 3, v.length
+      end
+
+    end
+  end
+
+  def test_traverse_flat_same
+    content =<<-EOF
+#Id    ValueA
+row1    a aa aaa
+row2    A
+row3    a
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :flat)
+      data = {}
+      k, f = tsv.traverse "Id", ["ValueA"] do |k,v|
+        data[k] = v
+      end
+      assert_equal %w(a aa aaa), data["row1"]
+    end
+  end
 end
 

data/test/scout/tsv/util/test_process.rb

@@ -63,5 +63,21 @@ row2    AA    BB    Id33
 
     assert_equal %w(), tsv["row1"]["ValueC"]
   end
+
+  def test_remove_duplicates
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|A|a|a    b|B|b|    Id1|Id2|Id1|Id1
+row2    aa|aa|AA|AA    b1|b2|B1|B2    Id1|Id1|Id2|Id2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+      assert_equal %w(a A a), tsv.remove_duplicates["row1"]["ValueA"]
+      assert tsv.remove_duplicates["row1"]["ValueB"].include?("")
+    end
+
+  end
+
 end
 

data/test/scout/tsv/util/test_reorder.rb

@@ -58,6 +58,23 @@ row2    A1
     assert_equal "row2", r["A1"]
   end
 
+  def test_reorder_single_to_double
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:single
+#Id    ValueA
+row1    a1
+row2    a1
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    r = tsv.reorder "ValueA", type: :flat
+
+    assert_equal ["row1", "row2"], r["a1"]
+  end
+
   def test_transpose
      content =<<-EOF
 #: :type=:list
@@ -90,5 +107,55 @@ rowa    a|aa    b|BB  C|CC
     end
   end
 
+  def test_slice_empty
+    content =<<-EOF
+#ID ValueA ValueB Comment
+row1 a b c
+row2 A B C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
+      tsv = tsv.slice []
+      assert tsv.fields.empty?
+      TmpFile.with_file do |tmpfile|
+        Open.write(tmpfile, tsv.to_s)
+        tsv = TSV.open tmpfile
+        assert tsv.fields.empty?
+      end
+    end
+  end
+
+  def test_reorder_flat
+    content =<<-EOF
+#Id    ValueA
+row1    a aa aaa
+row2    A
+row3    a
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :flat)
+
+      assert_equal ["row1", "row3"].sort, tsv.reorder("ValueA")["a"]
+    end
+  end
+
+  def test_reorder_flat_same
+    content =<<-EOF
+#Id    ValueA
+row1    a aa aaa
+row2    A
+row3    a
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :flat)
+
+      reordered = tsv.reorder("Id", ["ValueA"])
+      assert_equal %w(ValueA), reordered.fields
+      assert_equal ["a", "aa", "aaa"].sort, reordered["row1"]
+    end
+  end
 end
 

data/test/scout/tsv/util/test_sort.rb

@@ -5,7 +5,7 @@ require 'scout/tsv'
 
 class TestClass < Test::Unit::TestCase
 
-  def test_sort_by
+  def test_sort_by_empty
     content =<<-EOF
 #ID ValueA ValueB Comment
 row1 a B c
@@ -19,5 +19,32 @@ row2 A b C
     end
   end
 
+  def test_sort_by
+    content =<<-EOF
+#ID ValueA ValueB Comment
+row1 a B c
+row2 A b C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
+      assert_equal %w(row2 row1), tsv.sort_by("ValueA"){|k,v| v }.collect{|k,v| k}
+      assert_equal %w(row1 row2), tsv.sort_by("ValueB"){|k,v| v }.collect{|k,v| k}
+    end
+  end
+
+  def test_sort
+    content =<<-EOF
+#ID ValueA ValueB Comment
+row1 a B c
+row2 A b C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
+      assert_equal %w(row2 row1), tsv.sort("ValueA"){|a,b| a[1] <=> b[1] }.collect{|k,v| k}
+      assert_equal %w(row1 row2), tsv.sort("ValueB"){|a,b| a[1] <=> b[1] }.collect{|k,v| k}
+    end
+  end
 end
 

data/test/scout/tsv/util/test_unzip.rb

@@ -108,5 +108,37 @@ row2    A2|A22|A22    B2|B22|B22.2
     assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
     assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
   end
+
+  def test_unzip_replicates
+   content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b|bb|bbb    Id1|Id2|Id3
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+
+      assert_equal 4, tsv.unzip_replicates.length
+      assert_equal %w(aa bb Id2), tsv.unzip_replicates["row1(1)"]
+    end
+  end
+
+
+  def test_unzip_zip
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|A|a|a    b|B|b|    Id1|Id2|Id1|Id1
+row2    aa|aa|AA|AA    b1|b2|B1|B2    Id1|Id1|Id2|Id2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+      assert_equal ["b", "b", ""], tsv.unzip("ValueA", merge: true)["row1:a"]["ValueB"]
+      assert_equal ["b", "b", "", "B"].sort, tsv.unzip("ValueA", merge: true).zip(true)["row1"]["ValueB"].sort
+    end
+
+  end
+
 end
 

data/test/scout/work_queue/test_socket.rb

@@ -2,6 +2,9 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestSocket < Test::Unit::TestCase
+
+  class DoneProcessing end
+
   def test_simple
     socket = WorkQueue::Socket.new 
 
@@ -24,7 +27,7 @@ class TestSocket < Test::Unit::TestCase
   def __test_speed
     socket = WorkQueue::Socket.new 
 
-    num = 10_000
+    num = 50_000
 
     Thread.new do
       num.times do |i|

data/test/scout/workflow/deployment/test_orchestrator.rb

@@ -5,8 +5,9 @@ class TestOrchestrator < Test::Unit::TestCase
   setup do
     module TestWF
       extend Workflow 
+      self.name = "TestWF"
 
-      MULT = 0.01
+      MULT = 0.1
       task :a => :text do
         sleep(TestWF::MULT * (rand(10) + 2))
       end
@@ -34,11 +35,11 @@ class TestOrchestrator < Test::Unit::TestCase
     end
   end
 
-  def TODO_test_orchestrate_resources
+  def test_orchestrate_resources
 
     jobs =[]
 
-    num = 10
+    num = 5
     num.times do |i|
       jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
     end
@@ -64,16 +65,14 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 0 do
-      orchestrator.process(rules, jobs)
-    end
+    orchestrator = Workflow::Orchestrator.new(0.1, "cpus" => 30, "IO" => 10, "size" => 10 )
+    orchestrator.process(rules, jobs)
 
     data = Workflow.trace jobs, :plot_data => true
-    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    eend = data.column("End.second").values.collect{|v| v.to_f }.max
     second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
     (0..eend.to_i).each do |second|
-      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      tasks = data.select("Start.second"){|s| s <= second }.select("End.second"){|s| s > second}
       cpus = 0
       tasks.through :key, ["Workflow", "Task"] do |k, values|
         workflow, task = values
@@ -82,11 +81,11 @@ TestWF:
       second_cpus[second] = cpus
     end
 
-    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) > 15 
     assert Misc.mean(second_cpus.values) < 30
   end
 
-  def TODO_test_orchestrate_erase
+  def test_orchestrate_erase
 
     jobs =[]
 
@@ -119,9 +118,7 @@ TestWF:
     EOF
 
     orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 3 do
-      orchestrator.process(rules, jobs)
-    end
+    orchestrator.process(rules, jobs)
 
     jobs.each do |job|
       assert job.step(:c).dependencies.empty?
@@ -131,7 +128,7 @@ TestWF:
 
   end
 
-  def TODO_test_orchestrate_default
+  def test_orchestrate_default
 
     jobs =[]
 
@@ -166,9 +163,7 @@ TestWF:
     EOF
 
     orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 3 do
-      orchestrator.process(rules, jobs)
-    end
+    orchestrator.process(rules, jobs)
 
     jobs.each do |job|
       assert job.step(:c).dependencies.empty?
@@ -178,7 +173,7 @@ TestWF:
 
   end
 
-  def TODO_test_orchestrate_top_level
+  def test_orchestrate_top_level
 
     jobs =[]
 
@@ -211,9 +206,7 @@ TestWF:
     EOF
 
     orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 3 do
-      orchestrator.process(rules, jobs)
-    end
+    orchestrator.process(rules, jobs)
 
     jobs.each do |job|
       next unless job.task_name.to_s == 'd'
@@ -224,7 +217,7 @@ TestWF:
 
   end
 
-  def TODO_test_orchestrate_top_level_double_dep
+  def test_orchestrate_top_level_double_dep
 
     jobs =[]
 
@@ -257,9 +250,7 @@ TestWF:
     EOF
 
     orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 3 do
-      orchestrator.process(rules, jobs)
-    end
+    orchestrator.process(rules, jobs)
 
     jobs.each do |job|
       next unless job.task_name.to_s == 'd' || job.task_name.to_s == 'e'

data/test/scout/workflow/deployment/test_trace.rb

@@ -0,0 +1,25 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestWorkflowTrace < Test::Unit::TestCase
+  def test_true
+    m = Module.new do
+      extend Workflow
+      self.name = "TestWF"
+
+      input :option1
+      task :step1 do end
+
+      dep :step1
+      input :option2
+      task :step2 do end
+    end
+
+    job = m.job(:step2)
+    job.run
+    assert_equal 1, Workflow.trace([job])["TestWF#step1"]["Calls"]
+  end
+end
+

data/test/scout/workflow/step/test_archive.rb

@@ -0,0 +1,28 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestStepArchive < Test::Unit::TestCase
+  def test_archive
+    m = Module.new do
+      extend Workflow
+      self.name = "TestWF"
+
+      input :option1
+      task :step1 do end
+
+      dep :step1
+      input :option2
+      task :step2 do end
+    end
+
+    job = m.job(:step2, option1: "Option1", option2: "Option2")
+    job.run
+    job.archive_deps
+    assert_include job.archived_info, job.step(:step1).path
+    assert_equal :done, job.archived_info[job.step(:step1).path][:status]
+
+    assert_equal "Option1", job.archived_inputs[:option1]
+    assert_equal "Option1", job.inputs.concat(job.archived_inputs)[:option1]
+  end
+end
+

data/test/scout/workflow/step/test_children.rb

@@ -0,0 +1,25 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+
+class TestStepChildren < Test::Unit::TestCase
+  def test_child
+    TmpFile.with_file do |tmpfile|
+      step = Step.new tmpfile, ["12"] do |s|
+        pid = child do 
+          Open.write(self.file(:somefile), 'TEST')
+        end
+        Process.waitpid pid
+        s.length
+      end
+      step.type = :integer
+
+      assert_equal 2, step.run
+      assert_equal 1, step.info[:children_pids].length
+      assert_include step.files, 'somefile'
+    end
+  end
+
+end
+

data/test/scout/workflow/step/test_info.rb

@@ -42,4 +42,20 @@ class TestStepInfo < Test::Unit::TestCase
 
     end
   end
+
+  def test_messages
+    TmpFile.with_file do |tmpdir|
+      Path.setup(tmpdir)
+      tmpfile = tmpdir.test_step
+      step1 = Step.new tmpfile.step1, ["12"] do |s|
+        log :msg, "Message1"
+        log :msg, "Message2"
+        s.length
+      end
+
+      step1.run
+
+      assert_equal %w(Message1 Message2), step1.messages
+    end
+  end
 end

data/test/scout/workflow/task/test_dependencies.rb

@@ -244,8 +244,8 @@ class TestTaskDependencies < Test::Unit::TestCase
       task :step2 => :string do |i| step(:step1).load end
     end
 
-    job = wf.job(:step2, :input1 => 2)
-    assert_equal 1, job.run
+    job = wf.job(:step2, :input1 => "2")
+    assert_equal "1", job.run
     assert_equal Task::DEFAULT_NAME, job.name
     assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
   end
@@ -261,9 +261,9 @@ class TestTaskDependencies < Test::Unit::TestCase
       task :step2 => :string do |i| step(:step1).load end
     end
 
-    job = wf.job(:step2, "SOME_NAME", :input1 => 2)
+    job = wf.job(:step2, "SOME_NAME", :input1 => "2")
     assert_equal "SOME_NAME", job.step(:step1).clean_name
-    assert_equal 2, job.run
+    assert_equal "2", job.run
   end
 
   def test_override_inputs_block_array
@@ -278,8 +278,8 @@ class TestTaskDependencies < Test::Unit::TestCase
       task :step2 => :string do |i| step(:step1).load end
     end
 
-    job = wf.job(:step2, :input1 => 2)
-    assert_equal 1, job.run
+    job = wf.job(:step2, :input1 => "2")
+    assert_equal "1", job.run
     assert_equal Task::DEFAULT_NAME, job.name
     assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
   end
@@ -294,8 +294,8 @@ class TestTaskDependencies < Test::Unit::TestCase
       task :step2 => :string do |i| step(:step1).load end
     end
 
-    job = wf.job(:step2, :input1 => 2)
-    assert_equal 1, job.run
+    job = wf.job(:step2, :input1 => "2")
+    assert_equal "1", job.run
     assert_equal Task::DEFAULT_NAME, job.name
     assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
   end
@@ -313,13 +313,13 @@ class TestTaskDependencies < Test::Unit::TestCase
       task :step3 => :string do |i| step(:step1).load end
     end
 
-    job = wf.job(:step3, :input1 => 1)
+    job = wf.job(:step3, :input1 => "1")
     assert_equal Task::DEFAULT_NAME, job.name
     assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
-    assert_equal 2, job.run
+    assert_equal "2", job.run
 
-    job = wf.job(:step3, :input1 => 2)
-    assert_equal 3, job.run
+    job = wf.job(:step3, :input1 => "2")
+    assert_equal "3", job.run
 
 
     job = wf.job(:step3)
@@ -341,16 +341,16 @@ class TestTaskDependencies < Test::Unit::TestCase
       dep :step1, :canfail => true
       task :step2 => :string do |i|
         if step(:step1).error?
-          0
+          "0"
         else
           step(:step1).load 
         end
       end
     end
 
-    assert_equal 1, wf.job(:step2, :input1 => 1).run
-    assert_equal 2, wf.job(:step2, :input1 => 2).run
-    assert_equal 0, wf.job(:step2, :input1 => -2).run
+    assert_equal "1", wf.job(:step2, :input1 => "1").run
+    assert_equal "2", wf.job(:step2, :input1 => "2").run
+    assert_equal "0", wf.job(:step2, :input1 => "-2").run
   end
 
 end

data/test/scout/workflow/task/test_inputs.rb

@@ -1,7 +1,7 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-require 'scout/workflow/util'
+require 'scout/workflow'
 
 class TestTaskInput < Test::Unit::TestCase
   def example_workflow
@@ -227,4 +227,48 @@ class TestTaskInput < Test::Unit::TestCase
       end
     end
   end
+
+  def test_recursive_inputs
+    w = Module.new do
+      extend Workflow
+
+      self.name = "SaluteWF"
+
+      input :name, :string
+      task :salute => :string do |name|
+        "Hi #{name}"
+      end
+
+      task_alias :salute_miguel, self, :salute, :name => "Miguel"
+    end
+
+    assert w.tasks[:salute_miguel].recursive_inputs.empty?
+  end
+
+  def test_recursive_inputs_with_overriden_deps
+    w = Module.new do
+      extend Workflow
+
+      self.name = "SaluteWF"
+
+      input :name, :string
+      task :salute => :string do |name|
+        "Hi #{name}"
+      end
+
+      task_alias :salute_miguel, self, :salute, :name => "Miguel"
+
+      dep :salute
+      task :repeat_salute => :array do
+        [step(:salute).load] * 3
+      end
+
+      dep :salute_miguel
+      task_alias :repeat_salute_miguel, self, :repeat_salute, "SaluteWF#salute" => :salute_miguel
+    end
+
+    assert_equal ["Hi Miguel"] * 3, w.job(:repeat_salute_miguel).run
+
+    assert w.tasks[:repeat_salute_miguel].recursive_inputs.empty?
+  end
 end