scout-gear 7.3.0 → 8.1.0

data/test/scout/workflow/deployment/test_orchestrator.rb

@@ -0,0 +1,272 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOrchestrator < Test::Unit::TestCase
+  setup do
+    module TestWF
+      extend Workflow 
+
+      MULT = 0.01
+      task :a => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      task :b => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      dep :b
+      task :c => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :d => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :e => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+    end
+  end
+
+  def TODO_test_orchestrate_resources
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 0 do
+      orchestrator.process(rules, jobs)
+    end
+
+    data = Workflow.trace jobs, :plot_data => true
+    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
+    (0..eend.to_i).each do |second|
+      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      cpus = 0
+      tasks.through :key, ["Workflow", "Task"] do |k, values|
+        workflow, task = values
+        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
+      end
+      second_cpus[second] = cpus
+    end
+
+    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) < 30
+  end
+
+  def TODO_test_orchestrate_erase
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_default
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    erase: false
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_top_level
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd'
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_top_level_double_dep
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:e, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd' || job.task_name.to_s == 'e'
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+    end
+
+  end
+end
+

data/test/scout/workflow/step/test_dependencies.rb

@@ -21,5 +21,73 @@ class TestStepDependencies < Test::Unit::TestCase
     assert_equal 2, step2.inputs[:input2]
     assert_equal "12", step2.recursive_inputs[:input1]
   end
+
+  def test_can_fail
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, NamedArray.setup(["12"], %w(input1)) do |s|
+      raise ScoutException
+    end
+
+    step2 = Step.new tmpfile.step2, NamedArray.setup([2], %w(input2)) do |times|
+      step1 = dependencies.first
+      if step1.error?
+        (step1.inputs.first + " has unknown characters") * times
+      else
+        (step1.inputs.first + " has " + step1.load.to_s + " characters") * times
+      end
+    end
+
+    step2.dependencies = [step1]
+    step2.compute = {step1.path => [:canfail] }
+
+    assert_include step2.run, "unknown"
+  end
+
+  def test_can_stream
+    tmpfile = tmpdir.test_step
+
+    Misc.with_env "SCOUT_EXPLICIT_STREAMING", "true" do
+      times = 10_000
+      sleep = 1 / times
+
+      step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
+        Open.open_pipe do |sin|
+          times.times do |i|
+            sin.puts "line-#{i}"
+            sleep sleep
+          end
+        end
+      end
+      step1.type = :array
+
+      step2 = Step.new tmpfile.step2 do 
+        step1 = dependencies.first
+        raise ScoutException unless step1.streaming?
+        stream = step1.stream
+
+        Open.open_pipe do |sin|
+          while line = stream.gets
+            num = line.split("-").last
+            next if num.to_i % 2 == 1
+            sin.puts line
+          end
+        end
+      end
+      step2.type = :array
+      step2.dependencies = [step1]
+
+      assert_raise ScoutException do
+        step2.run
+      end
+
+      step2.recursive_clean
+
+      step2.compute = {step1.path => [:stream]}
+
+      assert_nothing_raised do
+        step2.run
+      end
+    end
+  end
 end
 

data/test/scout/workflow/step/test_info.rb

@@ -25,4 +25,22 @@ class TestStepInfo < Test::Unit::TestCase
       assert_equal "12 has 2 characters", step2.run
     end
   end
+
+  def test_inputs_marshal
+    TmpFile.with_file do |tmpdir|
+      Path.setup(tmpdir)
+      tmpfile = tmpdir.test_step
+
+      path = tmpfile.foo
+      step1 = Step.new tmpfile.step1, [path] do |s|
+        s.length
+      end
+
+      step1.run
+
+      refute Path === step1.info[:inputs][0]
+
+    end
+
+  end
 end

data/test/scout/workflow/step/test_status.rb

@@ -0,0 +1,30 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestStepStatus < Test::Unit::TestCase
+  def test_dependency
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
+
+    step2.dependencies = [step1]
+
+    Misc.with_env "SCOUT_UPDATE", "true" do
+      step2.run
+      assert step2.updated?
+
+      sleep 0.1
+      step1.clean
+      step1.run
+      refute step2.updated?
+    end
+  end
+
+end
+