scout-gear 8.0.0 → 9.0.0

data/test/scout/tsv/test_stream.rb

@@ -0,0 +1,200 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVStream < Test::Unit::TestCase
+  def test_paste_stream
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row1 A B C
+row2 AA BB CC
+row3 AAA BBB CCC
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb 
+row1 a b
+row2 aa bb
+row3 aaa bbb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row LabelC
+row1 c
+row2 cc
+row3 ccc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list)
+    assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_sort
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+row3 AAA BBB CCC
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb
+row1 a b
+row3 aaa bbb
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row1 c
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_missing_2
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "", "", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["", "", "", "", "", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_missing
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "", "", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["", "", "", "", "", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_missing_3
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labelc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.paste_streams([s1,s2], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labelc), tsv.fields
+    assert_equal ["A", "B", "C", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", ""], tsv["row2"]
+  end
+
+  def test_paste_stream_same_field
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA
+row1 A
+row2 AA
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row LabelA
+row2 AAA
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.paste_streams([s1,s2], :sep => " ", :type => :double, :sort => false, :same_fields => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal ["AA", "AAA"], tsv["row2"][0]
+  end
+
+  def test_paste_stream_nohead
+    text1=<<-EOF
+row1\tA
+row2\tAA
+    EOF
+
+    text2=<<-EOF
+row2\tAAA
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.paste_streams([s1,s2], :type => :double, :sort => false, :same_fields => true)
+    assert_equal ["AA", "AAA"], tsv["row2"][0]
+  end
+
+end
+

data/test/scout/tsv/test_transformer.rb

@@ -105,4 +105,16 @@ row2    A2|A22    B2|B22
     assert_equal "A1", tsv.to_single["row1"]
     assert_equal "A2", tsv.to_single["row2"]
   end
+
+  def test_to_flat
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TSV.open(content)
+    assert_equal %w(A1 A11 B1 B11), tsv.to_flat["row1"]
+  end
 end

data/test/scout/workflow/deployment/test_orchestrator.rb

@@ -0,0 +1,272 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOrchestrator < Test::Unit::TestCase
+  setup do
+    module TestWF
+      extend Workflow 
+
+      MULT = 0.01
+      task :a => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      task :b => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      dep :b
+      task :c => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :d => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :e => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+    end
+  end
+
+  def TODO_test_orchestrate_resources
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 0 do
+      orchestrator.process(rules, jobs)
+    end
+
+    data = Workflow.trace jobs, :plot_data => true
+    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
+    (0..eend.to_i).each do |second|
+      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      cpus = 0
+      tasks.through :key, ["Workflow", "Task"] do |k, values|
+        workflow, task = values
+        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
+      end
+      second_cpus[second] = cpus
+    end
+
+    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) < 30
+  end
+
+  def TODO_test_orchestrate_erase
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_default
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    erase: false
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_top_level
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd'
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_top_level_double_dep
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:e, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd' || job.task_name.to_s == 'e'
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+    end
+
+  end
+end
+

data/test/scout/workflow/step/test_dependencies.rb

@@ -21,5 +21,73 @@ class TestStepDependencies < Test::Unit::TestCase
     assert_equal 2, step2.inputs[:input2]
     assert_equal "12", step2.recursive_inputs[:input1]
   end
+
+  def test_can_fail
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, NamedArray.setup(["12"], %w(input1)) do |s|
+      raise ScoutException
+    end
+
+    step2 = Step.new tmpfile.step2, NamedArray.setup([2], %w(input2)) do |times|
+      step1 = dependencies.first
+      if step1.error?
+        (step1.inputs.first + " has unknown characters") * times
+      else
+        (step1.inputs.first + " has " + step1.load.to_s + " characters") * times
+      end
+    end
+
+    step2.dependencies = [step1]
+    step2.compute = {step1.path => [:canfail] }
+
+    assert_include step2.run, "unknown"
+  end
+
+  def test_can_stream
+    tmpfile = tmpdir.test_step
+
+    Misc.with_env "SCOUT_EXPLICIT_STREAMING", "true" do
+      times = 10_000
+      sleep = 1 / times
+
+      step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
+        Open.open_pipe do |sin|
+          times.times do |i|
+            sin.puts "line-#{i}"
+            sleep sleep
+          end
+        end
+      end
+      step1.type = :array
+
+      step2 = Step.new tmpfile.step2 do 
+        step1 = dependencies.first
+        raise ScoutException unless step1.streaming?
+        stream = step1.stream
+
+        Open.open_pipe do |sin|
+          while line = stream.gets
+            num = line.split("-").last
+            next if num.to_i % 2 == 1
+            sin.puts line
+          end
+        end
+      end
+      step2.type = :array
+      step2.dependencies = [step1]
+
+      assert_raise ScoutException do
+        step2.run
+      end
+
+      step2.recursive_clean
+
+      step2.compute = {step1.path => [:stream]}
+
+      assert_nothing_raised do
+        step2.run
+      end
+    end
+  end
 end
 

data/test/scout/workflow/step/test_info.rb

@@ -25,4 +25,21 @@ class TestStepInfo < Test::Unit::TestCase
       assert_equal "12 has 2 characters", step2.run
     end
   end
+
+  def test_inputs_marshal
+    TmpFile.with_file do |tmpdir|
+      Path.setup(tmpdir)
+      tmpfile = tmpdir.test_step
+
+      path = tmpfile.foo
+      step1 = Step.new tmpfile.step1, [path] do |s|
+        s.length
+      end
+
+      step1.run
+
+      refute Path === step1.info[:inputs][0]
+
+    end
+  end
 end

data/test/scout/workflow/step/test_status.rb

@@ -15,7 +15,6 @@ class TestStepStatus < Test::Unit::TestCase
 
     step2.dependencies = [step1]
 
-    sss 0
     Misc.with_env "SCOUT_UPDATE", "true" do
       step2.run
       assert step2.updated?