scout-gear 10.9.0 → 10.10.1

data/test/scout/workflow/deployment/{test_orchestrator.rb → test_local.rb}

@@ -1,38 +1,40 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-class TestOrchestrator < Test::Unit::TestCase
-  setup do
-    module TestWF
-      extend Workflow 
-      self.name = "TestWF"
+require 'scout/workflow/deployment/trace'
+module TestWF
+  extend Workflow 
+  self.name = "TestWF"
+
+  MULT = 0.1
+  task :a => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 
-      MULT = 0.1
-      task :a => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
+  dep :a
+  task :b => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 
-      dep :a
-      task :b => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
+  dep :a
+  dep :b
+  task :c => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 
-      dep :a
-      dep :b
-      task :c => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
+  dep :c
+  task :d => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 
-      dep :c
-      task :d => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
+  dep :c
+  task :e => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+end
 
-      dep :c
-      task :e => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
-    end
+class TestLocalExec < Test::Unit::TestCase
+  setup do
   end
 
   def test_orchestrate_resources
@@ -65,7 +67,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(0.1, "cpus" => 30, "IO" => 10, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(0.01, "cpus" => 30, "IO" => 10, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     data = Workflow.trace jobs, :plot_data => true
@@ -89,7 +91,50 @@ TestWF:
 
     jobs =[]
 
-    num = 10
+    num = 1
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator.process(rules, jobs)
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_erase_long
+
+    jobs =[]
+
+    num = 3
     num.times do |i|
       jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
     end
@@ -117,7 +162,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     jobs.each do |job|
@@ -162,7 +207,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     jobs.each do |job|
@@ -205,7 +250,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     jobs.each do |job|
@@ -249,7 +294,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     jobs.each do |job|
@@ -259,5 +304,88 @@ TestWF:
     end
 
   end
+
+  def test_orchestrate_produce
+
+    jobs =[]
+
+    num = 1
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    Workflow::LocalExecutor.produce(jobs, rules, produce_timer: 0.1, produce_cpus: 20)
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd'
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_produce_deps
+
+    jobs =[]
+
+    num = 1
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    Workflow::LocalExecutor.produce_dependencies(jobs, [:a, :b], rules, produce_timer: 0.1)
+
+    jobs.each do |job|
+      assert job.step(:a).done?
+      refute job.done?
+    end
+  end
 end
 

data/test/scout/workflow/deployment/test_scheduler.rb

@@ -0,0 +1,75 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow/deployment/scheduler/slurm'
+
+class TestScheduler < Test::Unit::TestCase
+  setup do
+    module TestWF
+      extend Workflow 
+      self.name = "TestWF"
+
+      MULT ||= 0.1
+      task :a => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      task :b => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      dep :b
+      task :c => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :d => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :e => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+    end
+  end
+
+  def test_orchestrate_process
+
+    jobs =[]
+
+    num = 1
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    batches = Workflow::Orchestrator.job_batches(rules, jobs)
+    dirs = Workflow::Scheduler.process_batches(batches, dry_run: true)
+    iii dirs
+  end
+end
+

data/test/scout/workflow/deployment/test_trace.rb

@@ -4,7 +4,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 require 'scout/tsv'
 
 class TestWorkflowTrace < Test::Unit::TestCase
-  def test_true
+  def test_trace
     m = Module.new do
       extend Workflow
       self.name = "TestWF"

data/test/scout/workflow/step/test_progress.rb

@@ -0,0 +1,27 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestStepProgress < Test::Unit::TestCase
+  def test_monitor_stream
+
+    TmpFile.with_file do |tmpfile|
+      items = %w(foo bar baz)
+      lines = []
+      step = Step.new tmpfile, ["12"] do |s|
+        file = file('test')
+        Open.write file, items * "\n"
+        s = file.open
+        self.monitor_stream s, bar: 3 do |line|
+          lines  << line.strip
+        end
+      end
+      step.type = :text
+
+      res = step.run(:stream)
+      res.read
+      res.join
+      assert_equal items, lines
+    end
+  end
+end
+

data/test/scout/workflow/task/test_inputs.rb

@@ -18,6 +18,7 @@ class TestTaskInput < Test::Unit::TestCase
       input :boolean_array, :boolean_array, "", [true, false, true]
       input :path_array, :path_array, "", %w(dir/subdir/file1 dir/subdir/file2)
       input :file_array, :file_array
+      input :select, :select, '', :foo, select_options: %(foo bar)
       task :task => :array do
         inputs
       end
@@ -124,6 +125,22 @@ class TestTaskInput < Test::Unit::TestCase
     end
   end
 
+  def test_save_and_load_select
+    task = self.example_task
+
+    TmpFile.with_file("2\n3") do |integer_array_file|
+      inputs = {:select => 'bar'}
+      original_digest =  task.process_inputs(inputs).last
+
+      TmpFile.with_path do |save_directory|
+        task.save_inputs(save_directory, inputs)
+        new_inputs = task.load_inputs(save_directory)
+        new_digest =  task.process_inputs(new_inputs).last
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
+
   def test_save_and_load_file
     task = self.example_task
 

data/test/test_helper.rb

@@ -51,11 +51,12 @@ class Test::Unit::TestCase
     Workflow.directory = tmpdir.var.jobs
     Workflow.workflows.each{|wf| wf.directory = Workflow.directory[wf.name] }
     Entity.entity_property_cache = tmpdir.entity_properties if defined?(Entity)
+    Workflow.job_cache.clear
+    SchedulerJob.batch_base_dir = tmpdir.batch
   end
   
   teardown do
     Open.rm_rf tmpdir
-    Workflow.job_cache.clear
   end
 
   def self.datadir_test

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.9.0
+  version: 10.10.1
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -189,8 +189,17 @@ files:
 - lib/scout/workflow.rb
 - lib/scout/workflow/definition.rb
 - lib/scout/workflow/deployment.rb
-- lib/scout/workflow/deployment/orchestrator.rb
+- lib/scout/workflow/deployment/local.rb
+- lib/scout/workflow/deployment/orchestrator/batches.rb
+- lib/scout/workflow/deployment/orchestrator/chains.rb
+- lib/scout/workflow/deployment/orchestrator/rules.rb
+- lib/scout/workflow/deployment/orchestrator/workload.rb
 - lib/scout/workflow/deployment/queue.rb
+- lib/scout/workflow/deployment/scheduler.rb
+- lib/scout/workflow/deployment/scheduler/job.rb
+- lib/scout/workflow/deployment/scheduler/lfs.rb
+- lib/scout/workflow/deployment/scheduler/pbs.rb
+- lib/scout/workflow/deployment/scheduler/slurm.rb
 - lib/scout/workflow/deployment/trace.rb
 - lib/scout/workflow/documentation.rb
 - lib/scout/workflow/entity.rb
@@ -320,13 +329,23 @@ files:
 - test/scout/tsv/util/test_unzip.rb
 - test/scout/work_queue/test_socket.rb
 - test/scout/work_queue/test_worker.rb
-- test/scout/workflow/deployment/test_orchestrator.rb
+- test/scout/workflow/deployment/orchestrator/test_batches.rb
+- test/scout/workflow/deployment/orchestrator/test_chains.rb
+- test/scout/workflow/deployment/orchestrator/test_rules.rb
+- test/scout/workflow/deployment/orchestrator/test_workload.rb
+- test/scout/workflow/deployment/scheduler/test_job.rb
+- test/scout/workflow/deployment/scheduler/test_lfs.rb
+- test/scout/workflow/deployment/scheduler/test_pbs.rb
+- test/scout/workflow/deployment/scheduler/test_slurm.rb
+- test/scout/workflow/deployment/test_local.rb
+- test/scout/workflow/deployment/test_scheduler.rb
 - test/scout/workflow/deployment/test_trace.rb
 - test/scout/workflow/step/test_archive.rb
 - test/scout/workflow/step/test_children.rb
 - test/scout/workflow/step/test_dependencies.rb
 - test/scout/workflow/step/test_info.rb
 - test/scout/workflow/step/test_load.rb
+- test/scout/workflow/step/test_progress.rb
 - test/scout/workflow/step/test_provenance.rb
 - test/scout/workflow/step/test_status.rb
 - test/scout/workflow/task/test_dependencies.rb