scout-gear 10.8.4 → 10.10.1

data/test/scout/workflow/deployment/scheduler/test_pbs.rb

@@ -0,0 +1,32 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+# Define a small workflow used by tests
+module TestSchedWF
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :b1 => :string do self.task_name.to_s end
+end
+
+class TestSchedulerJobPBS < Test::Unit::TestCase
+  def test_basic_template
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = PBS.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      assert_include tpl, 'module load java'
+      assert_include tpl, '#STEP_PATH'
+      assert_include tpl, '#PBS'
+    end
+  end
+
+  def test_singularity
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = PBS.job_template(job, :batch_dir => batch_dir, :singularity => true, :singularity_img => '/tmp/img.sif')
+      assert_include tpl, 'singularity exec'
+    end
+  end
+end

data/test/scout/workflow/deployment/scheduler/test_slurm.rb

@@ -0,0 +1,32 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+# Define a small workflow used by tests
+module TestSchedWF
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :b1 => :string do self.task_name.to_s end
+end
+
+class TestSchedulerJobSLURM < Test::Unit::TestCase
+  def test_basic_template
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = SLURM.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      assert_include tpl, 'module load java'
+      assert_include tpl, '#STEP_PATH'
+      assert_include tpl, '#SBATCH'
+    end
+  end
+
+  def test_singularity
+    job = TestSchedWF.job(:b1, "TEST")
+    TmpFile.with_file do |batch_dir|
+      tpl = SLURM.job_template(job, :batch_dir => batch_dir, :singularity => true, :singularity_img => '/tmp/img.sif')
+      assert_include tpl, 'singularity exec'
+    end
+  end
+end

data/test/scout/workflow/deployment/{test_orchestrator.rb → test_local.rb}

@@ -1,38 +1,40 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-class TestOrchestrator < Test::Unit::TestCase
-  setup do
-    module TestWF
-      extend Workflow 
-      self.name = "TestWF"
+require 'scout/workflow/deployment/trace'
+module TestWF
+  extend Workflow 
+  self.name = "TestWF"
+
+  MULT = 0.1
+  task :a => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 
-      MULT = 0.1
-      task :a => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
+  dep :a
+  task :b => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 
-      dep :a
-      task :b => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
+  dep :a
+  dep :b
+  task :c => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 
-      dep :a
-      dep :b
-      task :c => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
+  dep :c
+  task :d => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 
-      dep :c
-      task :d => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
+  dep :c
+  task :e => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+end
 
-      dep :c
-      task :e => :text do
-        sleep(TestWF::MULT * (rand(10) + 2))
-      end
-    end
+class TestLocalExec < Test::Unit::TestCase
+  setup do
   end
 
   def test_orchestrate_resources
@@ -65,7 +67,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(0.1, "cpus" => 30, "IO" => 10, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(0.01, "cpus" => 30, "IO" => 10, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     data = Workflow.trace jobs, :plot_data => true
@@ -89,7 +91,50 @@ TestWF:
 
     jobs =[]
 
-    num = 10
+    num = 1
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator.process(rules, jobs)
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_erase_long
+
+    jobs =[]
+
+    num = 3
     num.times do |i|
       jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
     end
@@ -117,7 +162,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     jobs.each do |job|
@@ -162,7 +207,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     jobs.each do |job|
@@ -205,7 +250,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     jobs.each do |job|
@@ -249,7 +294,7 @@ TestWF:
       cpus: 15
     EOF
 
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    orchestrator = Workflow::LocalExecutor.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
     orchestrator.process(rules, jobs)
 
     jobs.each do |job|
@@ -259,5 +304,88 @@ TestWF:
     end
 
   end
+
+  def test_orchestrate_produce
+
+    jobs =[]
+
+    num = 1
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    Workflow::LocalExecutor.produce(jobs, rules, produce_timer: 0.1, produce_cpus: 20)
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd'
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_produce_deps
+
+    jobs =[]
+
+    num = 1
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    Workflow::LocalExecutor.produce_dependencies(jobs, [:a, :b], rules, produce_timer: 0.1)
+
+    jobs.each do |job|
+      assert job.step(:a).done?
+      refute job.done?
+    end
+  end
 end
 

data/test/scout/workflow/deployment/test_scheduler.rb

@@ -0,0 +1,75 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow/deployment/scheduler/slurm'
+
+class TestScheduler < Test::Unit::TestCase
+  setup do
+    module TestWF
+      extend Workflow 
+      self.name = "TestWF"
+
+      MULT ||= 0.1
+      task :a => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      task :b => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      dep :b
+      task :c => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :d => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :e => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+    end
+  end
+
+  def test_orchestrate_process
+
+    jobs =[]
+
+    num = 1
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    batches = Workflow::Orchestrator.job_batches(rules, jobs)
+    dirs = Workflow::Scheduler.process_batches(batches, dry_run: true)
+    iii dirs
+  end
+end
+

data/test/scout/workflow/deployment/test_trace.rb

@@ -4,7 +4,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 require 'scout/tsv'
 
 class TestWorkflowTrace < Test::Unit::TestCase
-  def test_true
+  def test_trace
     m = Module.new do
       extend Workflow
       self.name = "TestWF"

data/test/scout/workflow/step/test_progress.rb

@@ -0,0 +1,27 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestStepProgress < Test::Unit::TestCase
+  def test_monitor_stream
+
+    TmpFile.with_file do |tmpfile|
+      items = %w(foo bar baz)
+      lines = []
+      step = Step.new tmpfile, ["12"] do |s|
+        file = file('test')
+        Open.write file, items * "\n"
+        s = file.open
+        self.monitor_stream s, bar: 3 do |line|
+          lines  << line.strip
+        end
+      end
+      step.type = :text
+
+      res = step.run(:stream)
+      res.read
+      res.join
+      assert_equal items, lines
+    end
+  end
+end
+

data/test/scout/workflow/task/test_inputs.rb

@@ -18,6 +18,7 @@ class TestTaskInput < Test::Unit::TestCase
       input :boolean_array, :boolean_array, "", [true, false, true]
       input :path_array, :path_array, "", %w(dir/subdir/file1 dir/subdir/file2)
       input :file_array, :file_array
+      input :select, :select, '', :foo, select_options: %(foo bar)
       task :task => :array do
         inputs
       end
@@ -124,6 +125,22 @@ class TestTaskInput < Test::Unit::TestCase
     end
   end
 
+  def test_save_and_load_select
+    task = self.example_task
+
+    TmpFile.with_file("2\n3") do |integer_array_file|
+      inputs = {:select => 'bar'}
+      original_digest =  task.process_inputs(inputs).last
+
+      TmpFile.with_path do |save_directory|
+        task.save_inputs(save_directory, inputs)
+        new_inputs = task.load_inputs(save_directory)
+        new_digest =  task.process_inputs(new_inputs).last
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
+
   def test_save_and_load_file
     task = self.example_task
 

data/test/test_helper.rb

@@ -51,11 +51,12 @@ class Test::Unit::TestCase
     Workflow.directory = tmpdir.var.jobs
     Workflow.workflows.each{|wf| wf.directory = Workflow.directory[wf.name] }
     Entity.entity_property_cache = tmpdir.entity_properties if defined?(Entity)
+    Workflow.job_cache.clear
+    SchedulerJob.batch_base_dir = tmpdir.batch
   end
   
   teardown do
     Open.rm_rf tmpdir
-    Workflow.job_cache.clear
   end
 
   def self.datadir_test

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.8.4
+  version: 10.10.1
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -101,18 +101,26 @@ executables:
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
+- README.md
 - README.rdoc
 files:
 - ".document"
 - ".gitmodules"
 - ".vimproject"
 - LICENSE.txt
+- README.md
 - README.rdoc
 - Rakefile
 - VERSION
 - bin/scout
-- doc/lib/scout/path.md
-- doc/lib/scout/workflow/task.md
+- doc/Association.md
+- doc/Entity.md
+- doc/KnowledgeBase.md
+- doc/Persist.md
+- doc/Semaphore.md
+- doc/TSV.md
+- doc/WorkQueue.md
+- doc/Workflow.md
 - lib/scout-gear.rb
 - lib/scout.rb
 - lib/scout/association.rb
@@ -181,8 +189,17 @@ files:
 - lib/scout/workflow.rb
 - lib/scout/workflow/definition.rb
 - lib/scout/workflow/deployment.rb
-- lib/scout/workflow/deployment/orchestrator.rb
+- lib/scout/workflow/deployment/local.rb
+- lib/scout/workflow/deployment/orchestrator/batches.rb
+- lib/scout/workflow/deployment/orchestrator/chains.rb
+- lib/scout/workflow/deployment/orchestrator/rules.rb
+- lib/scout/workflow/deployment/orchestrator/workload.rb
 - lib/scout/workflow/deployment/queue.rb
+- lib/scout/workflow/deployment/scheduler.rb
+- lib/scout/workflow/deployment/scheduler/job.rb
+- lib/scout/workflow/deployment/scheduler/lfs.rb
+- lib/scout/workflow/deployment/scheduler/pbs.rb
+- lib/scout/workflow/deployment/scheduler/slurm.rb
 - lib/scout/workflow/deployment/trace.rb
 - lib/scout/workflow/documentation.rb
 - lib/scout/workflow/entity.rb
@@ -213,7 +230,9 @@ files:
 - scout_commands/alias
 - scout_commands/batch/clean
 - scout_commands/batch/list
+- scout_commands/cat
 - scout_commands/doc
+- scout_commands/entity
 - scout_commands/find
 - scout_commands/glob
 - scout_commands/kb/config
@@ -228,6 +247,7 @@ files:
 - scout_commands/resource/produce
 - scout_commands/resource/sync
 - scout_commands/system/clean
+- scout_commands/system/status
 - scout_commands/template
 - scout_commands/update
 - scout_commands/workflow/cmd
@@ -309,13 +329,23 @@ files:
 - test/scout/tsv/util/test_unzip.rb
 - test/scout/work_queue/test_socket.rb
 - test/scout/work_queue/test_worker.rb
-- test/scout/workflow/deployment/test_orchestrator.rb
+- test/scout/workflow/deployment/orchestrator/test_batches.rb
+- test/scout/workflow/deployment/orchestrator/test_chains.rb
+- test/scout/workflow/deployment/orchestrator/test_rules.rb
+- test/scout/workflow/deployment/orchestrator/test_workload.rb
+- test/scout/workflow/deployment/scheduler/test_job.rb
+- test/scout/workflow/deployment/scheduler/test_lfs.rb
+- test/scout/workflow/deployment/scheduler/test_pbs.rb
+- test/scout/workflow/deployment/scheduler/test_slurm.rb
+- test/scout/workflow/deployment/test_local.rb
+- test/scout/workflow/deployment/test_scheduler.rb
 - test/scout/workflow/deployment/test_trace.rb
 - test/scout/workflow/step/test_archive.rb
 - test/scout/workflow/step/test_children.rb
 - test/scout/workflow/step/test_dependencies.rb
 - test/scout/workflow/step/test_info.rb
 - test/scout/workflow/step/test_load.rb
+- test/scout/workflow/step/test_progress.rb
 - test/scout/workflow/step/test_provenance.rb
 - test/scout/workflow/step/test_status.rb
 - test/scout/workflow/task/test_dependencies.rb
@@ -350,7 +380,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.7.0.dev
 specification_version: 4
 summary: basic gear for scouts
 test_files: []

data/doc/lib/scout/path.md

@@ -1,35 +0,0 @@
-Path
-===
-
-```ruby
-:current => '{PWD}/{TOPLEVEL}/{SUBPATH}',
-:user    => '{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
-:global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
-:scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
-:tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
-:default => :user
-
-@@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
-
-```
-
-
-
-# tags
-{PKGDIR}
-{LIBDIR}
-{RESOURCE}
-{HOME}
-{PWD}
-{TOPLEVEL}
-{SUBPATH}
-{BASENAME}
-{PATH}
-{MAPNAME}
-{REMOVE}

data/doc/lib/scout/workflow/task.md

@@ -1,13 +0,0 @@
-Task
-====
-
-```ruby
-    wf = Workflow.annonymous_workflow "TaskInputs" do
-      input :input1, :integer
-      task :step1 => :integer do |i| i end
-
-      dep :step1
-      input :input2, :integer, "Integer", 3
-      task :step2 => :integer do |i| i * step(:step1).load end
-    end
-```