scout-gear 10.9.0 → 10.10.1

data/lib/scout/workflow/task.rb

@@ -42,6 +42,8 @@ module Task
       memory_inputs = provided_inputs
     end
 
+    memory_inputs = nil if Array === memory_inputs && memory_inputs.compact.empty?
+    memory_inputs = nil if Hash === memory_inputs && memory_inputs.empty?
     Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {task: self.name, id: id, provided_inputs: memory_inputs}) do
       provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
       provided_inputs = {} if provided_inputs.nil?

data/scout-gear.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.9.0 ruby lib
+# stub: scout-gear 10.10.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.9.0".freeze
+  s.version = "10.10.1".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -106,8 +106,17 @@ Gem::Specification.new do |s|
     "lib/scout/workflow.rb",
     "lib/scout/workflow/definition.rb",
     "lib/scout/workflow/deployment.rb",
-    "lib/scout/workflow/deployment/orchestrator.rb",
+    "lib/scout/workflow/deployment/local.rb",
+    "lib/scout/workflow/deployment/orchestrator/batches.rb",
+    "lib/scout/workflow/deployment/orchestrator/chains.rb",
+    "lib/scout/workflow/deployment/orchestrator/rules.rb",
+    "lib/scout/workflow/deployment/orchestrator/workload.rb",
     "lib/scout/workflow/deployment/queue.rb",
+    "lib/scout/workflow/deployment/scheduler.rb",
+    "lib/scout/workflow/deployment/scheduler/job.rb",
+    "lib/scout/workflow/deployment/scheduler/lfs.rb",
+    "lib/scout/workflow/deployment/scheduler/pbs.rb",
+    "lib/scout/workflow/deployment/scheduler/slurm.rb",
     "lib/scout/workflow/deployment/trace.rb",
     "lib/scout/workflow/documentation.rb",
     "lib/scout/workflow/entity.rb",
@@ -237,13 +246,23 @@ Gem::Specification.new do |s|
     "test/scout/tsv/util/test_unzip.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
-    "test/scout/workflow/deployment/test_orchestrator.rb",
+    "test/scout/workflow/deployment/orchestrator/test_batches.rb",
+    "test/scout/workflow/deployment/orchestrator/test_chains.rb",
+    "test/scout/workflow/deployment/orchestrator/test_rules.rb",
+    "test/scout/workflow/deployment/orchestrator/test_workload.rb",
+    "test/scout/workflow/deployment/scheduler/test_job.rb",
+    "test/scout/workflow/deployment/scheduler/test_lfs.rb",
+    "test/scout/workflow/deployment/scheduler/test_pbs.rb",
+    "test/scout/workflow/deployment/scheduler/test_slurm.rb",
+    "test/scout/workflow/deployment/test_local.rb",
+    "test/scout/workflow/deployment/test_scheduler.rb",
     "test/scout/workflow/deployment/test_trace.rb",
     "test/scout/workflow/step/test_archive.rb",
     "test/scout/workflow/step/test_children.rb",
     "test/scout/workflow/step/test_dependencies.rb",
     "test/scout/workflow/step/test_info.rb",
     "test/scout/workflow/step/test_load.rb",
+    "test/scout/workflow/step/test_progress.rb",
     "test/scout/workflow/step/test_provenance.rb",
     "test/scout/workflow/step/test_status.rb",
     "test/scout/workflow/task/test_dependencies.rb",

data/scout_commands/batch/list

@@ -334,7 +334,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
       has_bar = false
-      (step.rec_dependencies + [step]).reverse.each do |j|
+      (step.rec_dependencies.to_a + [step]).reverse.each do |j|
         next if j.done? 
         if j.file(:progress).exists?
           bar = Log::ProgressBar.new 

data/scout_commands/workflow/cmd

@@ -12,20 +12,12 @@ $ #{$0} <workflow> <command> [<subcommands>] [<options>] [<arg> ...]
 
 -h--help Print this help
 EOF
-if options[:help]
-  if defined? scout_usage
-    scout_usage 
-  else
-    puts SOPT.doc
-  end
-  exit 0
-end
 
 workflow = ARGV.shift
 
 if workflow == '-h'
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -99,11 +91,11 @@ puts
 puts Log.color :magenta, "## COMMANDS"
 puts
 puts Log.color :magenta, "Command:"
-puts 
+puts
 puts " scout #{$previous_commands * " "} "
-puts 
+puts
 puts Log.color :magenta, "Subcommands:"
-puts 
+puts
 
 commands(prev).each do |command|
   directory = File.directory? dir[command].find
@@ -113,4 +105,4 @@ commands(prev).each do |command|
     puts "  " << command
   end
 end
-    
+

data/scout_commands/workflow/info

@@ -31,7 +31,7 @@ if options[:inputs]
   names, values = step.info.values_at :input_names, :inputs
   names.zip(values).each do |name,value|
     puts [Log.color(:title, name), Log.fingerprint(value)] * " = "
-  end
+  end if names
 elsif options[:recursive_inputs]
   step.recursive_inputs.each do |name,value|
     puts [Log.color(:title, name), Log.fingerprint(value)] * " = "

data/scout_commands/workflow/task

@@ -16,10 +16,13 @@ Run a workflow job
 $ #{$0} [<options>] <workflow> <task>
 
 -h--help Print this help
+--workflows* Other workflows to load
 --nostream Disable job streaming
 --update Update jobs with newer dependencies
 --deploy* Deploy mode: serial, local, queue, or SLURM (default 'serial')
+--deploy_rules* Deployment rules; can be files of keywords
 --fork Fork and return path
+--exec Exec and return output
 --load_inputs* Directory or file with inputs files to load
 --save_inputs* Directory or tar.gz file path to store inputs
 -jn--jobname* Name to use as job identifier
@@ -33,33 +36,45 @@ EOF
 
 workflow_name, task_name = ARGV
 
+
+help, workflows, provenance, clean, recursive_clean, clean_task, load_inputs, save_inputs, jobname, printpath, deploy, deploy_rules, override_deps, do_fork, do_exec = IndiferentHash.process_options options,
+  :help, :workflows, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :save_inputs, :jobname, :printpath, :deploy, :deploy_rules, :override_deps, :fork, :exec,
+  :deploy => 'serial'
+
 raise MissingParameterException.new :workflow if workflow_name.nil?
 
 workflow = Workflow.require_workflow workflow_name
+
+workflows = workflows.split(/,\s*/) if workflows
+workflows.each{|wf| Workflow.require_workflow wf } if workflows
+
 task_name = task_name.to_sym if task_name
 task = workflow.tasks[task_name.to_sym] if task_name
 
-options[:help] = true if task.nil?
-
-help, provenance, clean, recursive_clean, clean_task, load_inputs, save_inputs, jobname, printpath, deploy, override_deps, do_fork = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :save_inputs, :jobname, :printpath, :deploy, :override_deps, :fork,
-  :deploy => 'serial'
+help = true if task.nil?
 
 if help
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
 
   puts workflow.usage(task) if workflow
-  exit 0
+
+  if task_name && task.nil?
+    exit -1
+  else
+    exit 0
+  end
 end
 
+task = workflow.tasks[task_name]
+
 job_inputs = task.get_SOPT
 
 if load_inputs
-  job_inputs = job_inputs.merge(workflow.tasks[task_name].load_inputs(load_inputs))
+  job_inputs = job_inputs.merge(task.load_inputs(load_inputs))
 end
 
 if override_deps
@@ -99,10 +114,21 @@ elsif do_fork
   job.fork
   puts job.path
   exit 0
+elsif do_exec
+  puts job.exec
+  exit 0
 elsif save_inputs
   puts job.save_inputs(save_inputs)
   exit 0
 else
+  require 'scout/workflow/deployment'
+
+  if deploy_rules.nil?
+    rules = Workflow::Orchestrator.load_rules_for_job(job)
+  else
+    deploy_files = deploy_rules.split(/,\s*/)
+    rules = Workflow::Orchestrator.load_rules deploy_files
+  end
 
   case deploy
   when "queue"
@@ -112,28 +138,40 @@ else
       puts save_inputs
       exit
     end
+  when 'dry', 'dryrun', 'debug'
+    batches = Workflow::Orchestrator.job_batches rules, job
+    batches.each do |batch|
+      top = batch[:top_level]
+      jobs = batch[:jobs]
+      puts Log.color :title, Log.fingerprint(top)
+      puts batch[:rules].to_yaml
+      deps = batch[:deps].collect{|dep| dep[:top_level] }
+      provided_inputs = jobs.inject({}){|acc,j| acc = IndiferentHash.add_defaults acc, j.provided_inputs }
+      provided_inputs.each do |i,v|
+        puts Log.color :green, "#{i}: " + Log.color(:blue, Log.fingerprint(v))
+      end
+      other = jobs - [top]
+      puts Log.color :subtitle, 'Other jobs: ' + Log.color(:blue, other.collect{|d| d.short_path}.uniq * ', ' ) if other.any?
+      puts Log.color :subtitle, 'Dependencies: ' + Log.color(:blue, deps.collect{|d| d.short_path}.uniq * ', ' ) if deps.any?
+    end
+    exit 0
   when "serial"
     job.run(true) 
   when "local"
-    refresh = Scout::Config.get :refresh, :deploy, :local, :orchestrator, default: 0.5
-    orchestrator = Workflow::Orchestrator.new refresh.to_f, "cpus" => Misc.processors
-    orchestrator.process({}, job)
-  when "slurm"
-    require 'rbbt-scout'
-    require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
-    slurm_options = {}
-    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
-    HPC::BATCH_MODULE.orchestrate_job(job, slurm_options)
-    job.grace
-  else 
-    if deploy.end_with?('-slurm')
-      server = deploy.sub('-slurm','')
-      OffsiteStep.setup(job, server: server, slurm: true)
+    Workflow::LocalExecutor.produce job, rules
+  when 'batch', 'sched', 'cluster', "slurm", 'pbs', 'lsf'
+    Workflow::Scheduler.produce job, rules
+    exit 0
+  else
+    raise 'Use scout-camp instead of scout' unless defined? OffsiteStep
+    if deploy.end_with?('-batch')
+      server = deploy.sub('-batch','')
+      OffsiteStep.setup(job, server: server, batch: true)
     else
       OffsiteStep.setup(job, server: deploy)
     end
 
-    job.run
+    job.run(true)
   end unless job.done?
 
   if printpath
@@ -147,5 +185,3 @@ else
     end
   end
 end
-
-

data/test/scout/workflow/deployment/orchestrator/test_batches.rb

@@ -0,0 +1,138 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+  
+  dep_task :c4, TestWFC, :c3
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+module TestDeepWF
+  extend Workflow
+
+  dep_task :deep1, TestWFD, :d1
+
+  input :size, :integer, "Number of dependencies", 100
+  dep :deep1 do |jobname,options|
+    options[:size].to_i.times.collect{|i|
+      {:jobname => "step-#{i}"}
+    }
+  end
+  task :suite => :array do
+  end
+end
+
+class TestOrchestrateBatches < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+    time: 10min
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+    time: 10s
+  EOF
+
+  def test_chain_batches
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    job_chains_map = Workflow::Orchestrator.job_chains(RULES, job)
+    workload = Workflow::Orchestrator.job_workload(job)
+    batches = Workflow::Orchestrator.chain_batches(RULES, job_chains_map, workload)
+    assert_equal 3, batches.length
+  end
+
+
+  def test_job_batches_d
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    batches = Workflow::Orchestrator.job_batches(RULES, [job])
+    assert_equal 3, batches.length
+  end
+
+  def test_job_batches_c4
+    job = TestWFC.job(:c4, nil)
+    job.recursive_clean
+
+    batches = Workflow::Orchestrator.job_batches(RULES, job)
+    assert_equal 1, batches.length
+  end
+
+  def test_job_batches_a2
+    job = TestWFA.job(:a2, nil)
+    job.recursive_clean
+
+    batches = Workflow::Orchestrator.job_batches({}, job)
+    assert_equal 2, batches.length
+  end
+end

data/test/scout/workflow/deployment/orchestrator/test_chains.rb

@@ -0,0 +1,171 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+  
+  dep_task :c4, TestWFC, :c3
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+module TestDeepWF
+  extend Workflow
+
+  dep_task :deep1, TestWFD, :d1
+
+  input :size, :integer, "Number of dependencies", 100
+  dep :deep1 do |jobname,options|
+    options[:size].to_i.times.collect{|i|
+      {:jobname => "step-#{i}"}
+    }
+  end
+  task :suite => :array do
+  end
+end
+
+class TestOrchestrateChains < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    comment: Should not include TestWFA#a1
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+  EOF
+
+  def test_parse_chains
+    chains = Workflow::Orchestrator.parse_chains RULES
+    assert chains[:chain_a][:tasks]["TestWFA"].include?("a1")
+  end
+
+  def test_job_chain_a
+    job = TestWFA.job(:a3, nil)
+    job.recursive_clean
+
+    job_chains = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal %w(chain_a_b chain_a), job_chains.fields
+    assert_equal 3, job_chains["chain_a"].last[:jobs].length
+    assert_equal job, job_chains["chain_a"].last[:top_level]
+  end
+
+  def test_job_chain_b
+    job = TestWFB.job(:b2, nil)
+    job.recursive_clean
+
+    job_chains = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal %w(chain_a chain_b chain_b2 chain_a_b).sort, job_chains.fields.sort
+
+    assert_equal 2, job_chains["chain_a"].last[:jobs].length
+    assert_equal job.step("a2"), job_chains["chain_a"].last[:top_level]
+
+    assert_equal 2, job_chains["chain_b"].last[:jobs].length
+    assert_equal job, job_chains["chain_b"].last[:top_level]
+
+    assert_equal 2, job_chains["chain_b2"].last[:jobs].length
+    assert_equal job, job_chains["chain_b2"].last[:top_level]
+
+    assert_equal 4, job_chains["chain_a_b"].last[:jobs].length
+    assert_equal job, job_chains["chain_a_b"].last[:top_level]
+  end
+  
+  def test_job_chains_double
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    job_chains = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal 2, job_chains.select{|n,i| n == 'chain_a' }.length
+    assert_equal ["First c3", "Second c3"].sort, job_chains.
+      select{|n,i| n == 'chain_a' }.
+      collect{|n,i| i[:top_level].name }.sort
+
+    assert_equal 1, job_chains.select{|n,i| n == 'chain_d' }.length
+  end
+
+  def __test_benchmark
+
+    job = TestDeepWF.job(:suite, :size => 100)
+    job.recursive_clean
+
+    Misc.benchmark(10) do
+      Workflow::Orchestrator.job_chains(RULES, job)
+    end
+
+  end
+
+
+
+end
+