scout-gear 10.8.4 → 10.10.1

data/test/scout/workflow/deployment/orchestrator/test_batches.rb

@@ -0,0 +1,138 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+  
+  dep_task :c4, TestWFC, :c3
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+module TestDeepWF
+  extend Workflow
+
+  dep_task :deep1, TestWFD, :d1
+
+  input :size, :integer, "Number of dependencies", 100
+  dep :deep1 do |jobname,options|
+    options[:size].to_i.times.collect{|i|
+      {:jobname => "step-#{i}"}
+    }
+  end
+  task :suite => :array do
+  end
+end
+
+class TestOrchestrateBatches < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+    time: 10min
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+    time: 10s
+  EOF
+
+  def test_chain_batches
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    job_chains_map = Workflow::Orchestrator.job_chains(RULES, job)
+    workload = Workflow::Orchestrator.job_workload(job)
+    batches = Workflow::Orchestrator.chain_batches(RULES, job_chains_map, workload)
+    assert_equal 3, batches.length
+  end
+
+
+  def test_job_batches_d
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    batches = Workflow::Orchestrator.job_batches(RULES, [job])
+    assert_equal 3, batches.length
+  end
+
+  def test_job_batches_c4
+    job = TestWFC.job(:c4, nil)
+    job.recursive_clean
+
+    batches = Workflow::Orchestrator.job_batches(RULES, job)
+    assert_equal 1, batches.length
+  end
+
+  def test_job_batches_a2
+    job = TestWFA.job(:a2, nil)
+    job.recursive_clean
+
+    batches = Workflow::Orchestrator.job_batches({}, job)
+    assert_equal 2, batches.length
+  end
+end

data/test/scout/workflow/deployment/orchestrator/test_chains.rb

@@ -0,0 +1,171 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+  
+  dep_task :c4, TestWFC, :c3
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+module TestDeepWF
+  extend Workflow
+
+  dep_task :deep1, TestWFD, :d1
+
+  input :size, :integer, "Number of dependencies", 100
+  dep :deep1 do |jobname,options|
+    options[:size].to_i.times.collect{|i|
+      {:jobname => "step-#{i}"}
+    }
+  end
+  task :suite => :array do
+  end
+end
+
+class TestOrchestrateChains < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    comment: Should not include TestWFA#a1
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+  EOF
+
+  def test_parse_chains
+    chains = Workflow::Orchestrator.parse_chains RULES
+    assert chains[:chain_a][:tasks]["TestWFA"].include?("a1")
+  end
+
+  def test_job_chain_a
+    job = TestWFA.job(:a3, nil)
+    job.recursive_clean
+
+    job_chains = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal %w(chain_a_b chain_a), job_chains.fields
+    assert_equal 3, job_chains["chain_a"].last[:jobs].length
+    assert_equal job, job_chains["chain_a"].last[:top_level]
+  end
+
+  def test_job_chain_b
+    job = TestWFB.job(:b2, nil)
+    job.recursive_clean
+
+    job_chains = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal %w(chain_a chain_b chain_b2 chain_a_b).sort, job_chains.fields.sort
+
+    assert_equal 2, job_chains["chain_a"].last[:jobs].length
+    assert_equal job.step("a2"), job_chains["chain_a"].last[:top_level]
+
+    assert_equal 2, job_chains["chain_b"].last[:jobs].length
+    assert_equal job, job_chains["chain_b"].last[:top_level]
+
+    assert_equal 2, job_chains["chain_b2"].last[:jobs].length
+    assert_equal job, job_chains["chain_b2"].last[:top_level]
+
+    assert_equal 4, job_chains["chain_a_b"].last[:jobs].length
+    assert_equal job, job_chains["chain_a_b"].last[:top_level]
+  end
+  
+  def test_job_chains_double
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    job_chains = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal 2, job_chains.select{|n,i| n == 'chain_a' }.length
+    assert_equal ["First c3", "Second c3"].sort, job_chains.
+      select{|n,i| n == 'chain_a' }.
+      collect{|n,i| i[:top_level].name }.sort
+
+    assert_equal 1, job_chains.select{|n,i| n == 'chain_d' }.length
+  end
+
+  def __test_benchmark
+
+    job = TestDeepWF.job(:suite, :size => 100)
+    job.recursive_clean
+
+    Misc.benchmark(10) do
+      Workflow::Orchestrator.job_chains(RULES, job)
+    end
+
+  end
+
+
+
+end
+

data/test/scout/workflow/deployment/orchestrator/test_rules.rb

@@ -0,0 +1,219 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOrchestrateRules < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    comment: This chain is not valid, it is missing a2
+    tasks: TestWFA#a1, TestWFA#a3, TestWFB#b1, TestWFB#b2
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+  EOF
+
+  def test_defaults
+    rules = Workflow::Orchestrator.task_specific_rules RULES, "TestWFA", :a1
+    assert_equal "first_queue" , rules[:queue]
+  end
+
+
+  def test_task_options
+    rules = Workflow::Orchestrator.task_specific_rules RULES, "TestWFA", :a1
+    assert_equal 10, rules[:cpus]
+    assert_equal 4, rules[:log]
+  end
+
+  def test_skip
+    rules = Workflow::Orchestrator.task_specific_rules RULES, "TestWFC", :c1
+    assert rules[:skip]
+  end
+
+  def test_load_rules
+    rules1 =<<-EOF
+defaults:
+  name: rules1
+  time: 1h
+  cpus: 2
+Workflow:
+  defaults:
+    time: 1m
+    cpus: 4
+Workflow3:
+  defaults:
+    time: 3m
+    EOF
+
+    rules2 =<<-EOF
+defaults:
+  name: rules2
+  time: 2h
+Workflow:
+  defaults:
+    time: 2m
+Workflow2:
+  defaults:
+    time: 3m
+    EOF
+
+    TmpFile.with_file rules1 do |file1|
+      TmpFile.with_file rules2 do |file2|
+
+        rules = Workflow::Orchestrator.load_rules [file2, file1]
+
+        assert_equal 'rules2', rules[:defaults][:name]
+
+        assert_equal 2, rules[:defaults][:cpus]
+        assert_equal '2h', rules[:defaults][:time]
+
+        assert_equal 4, rules['Workflow'][:defaults][:cpus]
+        assert_equal '2m', rules["Workflow"][:defaults][:time]
+
+        assert_include rules, 'Workflow2'
+        assert_include rules, 'Workflow3'
+      end
+    end
+  end
+
+  def test_load_rules_config_keys
+    rules1 =<<-EOF
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 2
+    config_keys: cpus 4 loop
+    EOF
+
+    rules2 =<<-EOF
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 4
+    config_keys: mem high database
+    EOF
+
+    correct = YAML.load <<-EOF
+---
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 4
+    config_keys: cpus 4 loop,mem high database
+    EOF
+
+
+    TmpFile.with_file rules1 do |file1|
+      TmpFile.with_file rules2 do |file2|
+
+        rules = Workflow::Orchestrator.load_rules [file2, file1]
+        assert_equal correct, rules
+      end
+    end
+  end
+
+
+  def test_load_rules_config_keys_alt
+    rules1 =<<-EOF
+Workflow:
+  defaults:
+    name: rules1
+    time: 1h
+    cpus: 2
+    config_keys:
+      - cpus 4 loop
+      - level warn log
+    EOF
+
+    rules2 =<<-EOF
+Workflow:
+  defaults:
+    name: rules2
+    time: 1h
+    cpus: 4
+    config_keys:
+      - mem high database
+    EOF
+
+    correct = YAML.load <<-EOF
+---
+Workflow:
+  defaults:
+    name: rules2
+    time: 1h
+    cpus: 4
+    config_keys: cpus 4 loop,level warn log,mem high database
+    EOF
+
+
+    TmpFile.with_file rules1 do |file1|
+      TmpFile.with_file rules2 do |file2|
+
+        rules = Workflow::Orchestrator.load_rules [file2, file1]
+        assert_equal correct, rules
+      end
+    end
+  end
+
+
+  def test_load_rules_import
+    rules1 =<<-EOF
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 2
+    config_keys: cpus 4 loop
+    EOF
+
+    rules2 =<<-EOF
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 2
+    config_keys: mem high database
+    EOF
+
+    correct = YAML.load <<-EOF
+---
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 2
+    config_keys: cpus 4 loop,mem high database
+    EOF
+
+
+    TmpFile.with_file rules1 do |file1|
+      TmpFile.with_file rules2 do |file2|
+        Open.open(file2, mode: 'a') do |f|
+          f.puts "import: #{file1}"
+        end
+
+        rules = Workflow::Orchestrator.load_rules file2
+        assert_equal correct, rules
+      end
+    end
+  end
+end
+

data/test/scout/workflow/deployment/orchestrator/test_workload.rb

@@ -0,0 +1,117 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+  
+  dep_task :c4, TestWFC, :c3
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+module TestDeepWF
+  extend Workflow
+
+  dep_task :deep1, TestWFD, :d1
+
+  input :size, :integer, "Number of dependencies", 100
+  dep :deep1 do |jobname,options|
+    options[:size].to_i.times.collect{|i|
+      {:jobname => "step-#{i}"}
+    }
+  end
+  task :suite => :array do
+  end
+end
+
+class TestWorkload < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+    time: 10min
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+    time: 10s
+  EOF
+
+  def test_workload
+    job = TestWFC.job(:c4, nil)
+    job.recursive_clean
+
+    workload = Workflow::Orchestrator.job_workload(job)
+    assert_equal [], workload[job.step(:c2)]
+    assert workload.select{|job,deps| deps.include? job }.empty?
+    assert_include workload[job], job.dependencies.first
+
+    workload = Workflow::Orchestrator.job_workload([job])
+    assert_equal [], workload[job.step(:c2)]
+    assert workload.select{|job,deps| deps.include? job }.empty?
+    assert_include workload[job], job.dependencies.first
+  end
+end

data/test/scout/workflow/deployment/scheduler/test_job.rb

@@ -0,0 +1,31 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+# Define a small workflow used by tests
+module TestSchedWF
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :b1 => :string do self.task_name.to_s end
+end
+
+class TestSchedulerJob < Test::Unit::TestCase
+  def test_basic_template
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = SchedulerJob.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      assert_include tpl, 'module load java'
+      assert_include tpl, '#STEP_PATH'
+    end
+  end
+
+  def test_singularity
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = SchedulerJob.job_template(job, :batch_dir => batch_dir, :singularity => true, :singularity_img => '/tmp/img.sif')
+      assert_include tpl, 'singularity exec'
+    end
+  end
+end

data/test/scout/workflow/deployment/scheduler/test_lfs.rb

@@ -0,0 +1,32 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+# Define a small workflow used by tests
+module TestSchedWF
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :b1 => :string do self.task_name.to_s end
+end
+
+class TestSchedulerJobLSF < Test::Unit::TestCase
+  def test_basic_template
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = LSF.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      assert_include tpl, 'module load java'
+      assert_include tpl, '#STEP_PATH'
+      assert_include tpl, '#BSUB'
+    end
+  end
+
+  def test_singularity
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = LSF.job_template(job, :batch_dir => batch_dir, :singularity => true, :singularity_img => '/tmp/img.sif')
+      assert_include tpl, 'singularity exec'
+    end
+  end
+end