scout-gear 10.9.0 → 10.10.1

data/test/scout/workflow/deployment/orchestrator/test_rules.rb

@@ -0,0 +1,219 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOrchestrateRules < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    comment: This chain is not valid, it is missing a2
+    tasks: TestWFA#a1, TestWFA#a3, TestWFB#b1, TestWFB#b2
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+  EOF
+
+  def test_defaults
+    rules = Workflow::Orchestrator.task_specific_rules RULES, "TestWFA", :a1
+    assert_equal "first_queue" , rules[:queue]
+  end
+
+
+  def test_task_options
+    rules = Workflow::Orchestrator.task_specific_rules RULES, "TestWFA", :a1
+    assert_equal 10, rules[:cpus]
+    assert_equal 4, rules[:log]
+  end
+
+  def test_skip
+    rules = Workflow::Orchestrator.task_specific_rules RULES, "TestWFC", :c1
+    assert rules[:skip]
+  end
+
+  def test_load_rules
+    rules1 =<<-EOF
+defaults:
+  name: rules1
+  time: 1h
+  cpus: 2
+Workflow:
+  defaults:
+    time: 1m
+    cpus: 4
+Workflow3:
+  defaults:
+    time: 3m
+    EOF
+
+    rules2 =<<-EOF
+defaults:
+  name: rules2
+  time: 2h
+Workflow:
+  defaults:
+    time: 2m
+Workflow2:
+  defaults:
+    time: 3m
+    EOF
+
+    TmpFile.with_file rules1 do |file1|
+      TmpFile.with_file rules2 do |file2|
+
+        rules = Workflow::Orchestrator.load_rules [file2, file1]
+
+        assert_equal 'rules2', rules[:defaults][:name]
+
+        assert_equal 2, rules[:defaults][:cpus]
+        assert_equal '2h', rules[:defaults][:time]
+
+        assert_equal 4, rules['Workflow'][:defaults][:cpus]
+        assert_equal '2m', rules["Workflow"][:defaults][:time]
+
+        assert_include rules, 'Workflow2'
+        assert_include rules, 'Workflow3'
+      end
+    end
+  end
+
+  def test_load_rules_config_keys
+    rules1 =<<-EOF
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 2
+    config_keys: cpus 4 loop
+    EOF
+
+    rules2 =<<-EOF
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 4
+    config_keys: mem high database
+    EOF
+
+    correct = YAML.load <<-EOF
+---
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 4
+    config_keys: cpus 4 loop,mem high database
+    EOF
+
+
+    TmpFile.with_file rules1 do |file1|
+      TmpFile.with_file rules2 do |file2|
+
+        rules = Workflow::Orchestrator.load_rules [file2, file1]
+        assert_equal correct, rules
+      end
+    end
+  end
+
+
+  def test_load_rules_config_keys_alt
+    rules1 =<<-EOF
+Workflow:
+  defaults:
+    name: rules1
+    time: 1h
+    cpus: 2
+    config_keys:
+      - cpus 4 loop
+      - level warn log
+    EOF
+
+    rules2 =<<-EOF
+Workflow:
+  defaults:
+    name: rules2
+    time: 1h
+    cpus: 4
+    config_keys:
+      - mem high database
+    EOF
+
+    correct = YAML.load <<-EOF
+---
+Workflow:
+  defaults:
+    name: rules2
+    time: 1h
+    cpus: 4
+    config_keys: cpus 4 loop,level warn log,mem high database
+    EOF
+
+
+    TmpFile.with_file rules1 do |file1|
+      TmpFile.with_file rules2 do |file2|
+
+        rules = Workflow::Orchestrator.load_rules [file2, file1]
+        assert_equal correct, rules
+      end
+    end
+  end
+
+
+  def test_load_rules_import
+    rules1 =<<-EOF
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 2
+    config_keys: cpus 4 loop
+    EOF
+
+    rules2 =<<-EOF
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 2
+    config_keys: mem high database
+    EOF
+
+    correct = YAML.load <<-EOF
+---
+Workflow:
+  defaults:
+    time: 1h
+    cpus: 2
+    config_keys: cpus 4 loop,mem high database
+    EOF
+
+
+    TmpFile.with_file rules1 do |file1|
+      TmpFile.with_file rules2 do |file2|
+        Open.open(file2, mode: 'a') do |f|
+          f.puts "import: #{file1}"
+        end
+
+        rules = Workflow::Orchestrator.load_rules file2
+        assert_equal correct, rules
+      end
+    end
+  end
+end
+

data/test/scout/workflow/deployment/orchestrator/test_workload.rb

@@ -0,0 +1,117 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+  
+  dep_task :c4, TestWFC, :c3
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+module TestDeepWF
+  extend Workflow
+
+  dep_task :deep1, TestWFD, :d1
+
+  input :size, :integer, "Number of dependencies", 100
+  dep :deep1 do |jobname,options|
+    options[:size].to_i.times.collect{|i|
+      {:jobname => "step-#{i}"}
+    }
+  end
+  task :suite => :array do
+  end
+end
+
+class TestWorkload < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+    time: 10min
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+    time: 10s
+  EOF
+
+  def test_workload
+    job = TestWFC.job(:c4, nil)
+    job.recursive_clean
+
+    workload = Workflow::Orchestrator.job_workload(job)
+    assert_equal [], workload[job.step(:c2)]
+    assert workload.select{|job,deps| deps.include? job }.empty?
+    assert_include workload[job], job.dependencies.first
+
+    workload = Workflow::Orchestrator.job_workload([job])
+    assert_equal [], workload[job.step(:c2)]
+    assert workload.select{|job,deps| deps.include? job }.empty?
+    assert_include workload[job], job.dependencies.first
+  end
+end

data/test/scout/workflow/deployment/scheduler/test_job.rb

@@ -0,0 +1,31 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+# Define a small workflow used by tests
+module TestSchedWF
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :b1 => :string do self.task_name.to_s end
+end
+
+class TestSchedulerJob < Test::Unit::TestCase
+  def test_basic_template
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = SchedulerJob.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      assert_include tpl, 'module load java'
+      assert_include tpl, '#STEP_PATH'
+    end
+  end
+
+  def test_singularity
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = SchedulerJob.job_template(job, :batch_dir => batch_dir, :singularity => true, :singularity_img => '/tmp/img.sif')
+      assert_include tpl, 'singularity exec'
+    end
+  end
+end

data/test/scout/workflow/deployment/scheduler/test_lfs.rb

@@ -0,0 +1,32 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+# Define a small workflow used by tests
+module TestSchedWF
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :b1 => :string do self.task_name.to_s end
+end
+
+class TestSchedulerJobLSF < Test::Unit::TestCase
+  def test_basic_template
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = LSF.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      assert_include tpl, 'module load java'
+      assert_include tpl, '#STEP_PATH'
+      assert_include tpl, '#BSUB'
+    end
+  end
+
+  def test_singularity
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = LSF.job_template(job, :batch_dir => batch_dir, :singularity => true, :singularity_img => '/tmp/img.sif')
+      assert_include tpl, 'singularity exec'
+    end
+  end
+end

data/test/scout/workflow/deployment/scheduler/test_pbs.rb

@@ -0,0 +1,32 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+# Define a small workflow used by tests
+module TestSchedWF
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :b1 => :string do self.task_name.to_s end
+end
+
+class TestSchedulerJobPBS < Test::Unit::TestCase
+  def test_basic_template
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = PBS.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      assert_include tpl, 'module load java'
+      assert_include tpl, '#STEP_PATH'
+      assert_include tpl, '#PBS'
+    end
+  end
+
+  def test_singularity
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = PBS.job_template(job, :batch_dir => batch_dir, :singularity => true, :singularity_img => '/tmp/img.sif')
+      assert_include tpl, 'singularity exec'
+    end
+  end
+end

data/test/scout/workflow/deployment/scheduler/test_slurm.rb

@@ -0,0 +1,32 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+# Define a small workflow used by tests
+module TestSchedWF
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :b1 => :string do self.task_name.to_s end
+end
+
+class TestSchedulerJobSLURM < Test::Unit::TestCase
+  def test_basic_template
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = SLURM.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      assert_include tpl, 'module load java'
+      assert_include tpl, '#STEP_PATH'
+      assert_include tpl, '#SBATCH'
+    end
+  end
+
+  def test_singularity
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = SLURM.job_template(job, :batch_dir => batch_dir, :singularity => true, :singularity_img => '/tmp/img.sif')
+      assert_include tpl, 'singularity exec'
+    end
+  end
+end