scout-gear 10.9.0 → 10.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3900d5a0e546d494ae3224176db5feb92344dc1f6ba311461c74d02f5b137a9c
-  data.tar.gz: 35421139fea183944db794bad361bdfa9db78b086e13fab6367d6bbcfebc29e3
+  metadata.gz: f2debf30c7757ca3bd5ce191b99ce173bc31ea86e814933f4f7589a83ef49517
+  data.tar.gz: 6ad6353a5e9eeade8169c76d3f0b1501f8f9d02fa1e826538e15e152e79582ad
 SHA512:
-  metadata.gz: 679d05d7ae724825e18a2ce12862d08a3e75179f575a19ff960f4f379be6f2ce844326beca6e47d4bce3f585a4e637c71164b8642a2d61a85b2a54236f345153
-  data.tar.gz: c952339dfaa42d948a297a7f2c456d63bb4624d0ec71ee26499b79a2a25be8f32ec8c2b5ea2bd9cc17a664c50edb8f6cf22dec458ea103eb6e82614774973bdc
+  metadata.gz: bead6f4347748fc7365d6e24da5259417a8f3ea802ae5dbfaebc1056af062fb7af0b463eed3fe9669c7321e701daaf0fcf04245563a288feb682f9e2ff54ef29
+  data.tar.gz: 20874d977a84fab65c3c3bc50c96d48976a577f697d8e8db22b83de49b576639d1bae93a6ce604bb34cfdd86ed0fbf24baadd212077992797c2f955162139623

data/.vimproject

@@ -2,6 +2,10 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
  Rakefile
  README.md
  chats=chats filter="*"{
+
+  deploy
+
+
   debug
   document.rb
   pipes
@@ -46,10 +50,31 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      inputs.rb
      info.rb
     }
+
     deployment.rb
     deployment=deployment{
+     orchestrator=orchestrator{
+      rules.rb
+      workload.rb
+      chains.rb
+      batches.rb
+     }
+
+     local.rb
+
+     scheduler.rb
+
+     scheduler=scheduler{
+      job.rb
+      lfs.rb
+      slurm.rb
+      pbs.rb
+     }
+
      queue.rb
+
      trace.rb
+
      orchestrator.rb
     }
     entity.rb

data/VERSION

@@ -1 +1 @@
-10.9.0
+10.10.1

data/bin/scout

@@ -90,7 +90,7 @@ end
 if options[:log]
   Log.severity = options.delete(:log).to_i
   require 'scout-gear'
-else
+elsif Log.severity.nil?
   require 'scout-gear'
 
   global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
@@ -100,8 +100,11 @@ else
     global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
     Log.severity = global_severity.to_i if global_severity
   end
+else
+  require 'scout-gear'
 end
 
+
 if config_keys = options.delete(:config_keys)
   config_keys.split(",").each do |config|
     config = config.strip

data/lib/scout/knowledge_base/registry.rb

@@ -175,14 +175,13 @@ class KnowledgeBase
 
           database = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
                        Log.low "Re-opening database #{ name } from #{ Log.fingerprint persist_path }. #{options}"
-                       Association.database(file, **options.merge(persist_options: persist_options))
+                       Association.database(file, **options.merge(persist_options: persist_options).except(:undirected))
                      else
                        options = IndiferentHash.add_defaults options, registered_options if registered_options
-                       undirected = IndiferentHash.process_options options, :undirected
                        raise "Repo #{ name } not found and not registered" if file.nil?
                        Log.medium "Opening database #{ name } from #{ Log.fingerprint file }. #{options}"
                        file = file.call if Proc === file
-                       Association.database(file, **options.merge(persist_options: persist_options))
+                       Association.database(file, **options.merge(persist_options: persist_options).except(:undirected))
                      end
 
           database.namespace = self.namespace if self.namespace

data/lib/scout/workflow/definition.rb

@@ -227,4 +227,15 @@ module Workflow
   alias export_asynchronous export
   alias export_exec export
   alias export_stream export
+
+  def include_workflow(workflow)
+    workflow.documentation
+    self.asynchronous_exports += workflow.asynchronous_exports
+    self.synchronous_exports += workflow.synchronous_exports
+    self.exec_exports += workflow.exec_exports
+    self.stream_exports += workflow.stream_exports
+    self.tasks.merge! workflow.tasks
+    self.tasks.each{|_,t| t.workflow = workflow }
+    self.helpers.merge! workflow.helpers
+  end
 end

data/lib/scout/workflow/deployment/local.rb

@@ -0,0 +1,288 @@
+require_relative 'orchestrator/batches'
+class Workflow::LocalExecutor
+  class NoWork < Exception; end
+
+  def self.process(*args)
+    self.new.process(*args)
+  end
+
+  def self.produce(jobs, rules = {}, produce_cpus: Etc.nprocessors, produce_timer: 1)
+    jobs = [jobs] unless Array === jobs
+    orchestrator = self.new produce_timer.to_f, cpus: produce_cpus.to_i
+    begin
+      orchestrator.process(rules, jobs)
+    rescue self::NoWork
+    end
+  end
+
+  def self.produce_dependencies(jobs, tasks, rules = {}, produce_cpus: Etc.nprocessors, produce_timer: 1)
+    jobs = [jobs] unless Array === jobs
+    tasks = tasks.collect{|task| (String === task) ? task.to_sym : task }
+
+    produce_list = []
+    jobs.each do |job|
+      next if job.done? || job.running?
+      job.rec_dependencies.each do |dep|
+        task_name = dep.task_name.to_sym
+        task_name = task_name.to_sym if String === task_name
+        produce_list << dep if tasks.include?(task_name) ||
+          tasks.include?(job.task_name.to_s) ||
+          tasks.include?(job.full_task_name)
+      end
+    end
+
+    produce(produce_list, rules, produce_cpus: produce_cpus, produce_timer: produce_timer)
+  end
+
+  attr_accessor :available_resources, :resources_requested, :resources_used, :timer
+
+  def initialize(timer = 5, available_resources = nil)
+    available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
+    @timer               = timer
+    @available_resources = IndiferentHash.setup(available_resources)
+    @resources_requested = IndiferentHash.setup({})
+    @resources_used      = IndiferentHash.setup({})
+  end
+
+  def process_batches(batches)
+    failed_jobs = []
+
+    while batches.reject{|b| Workflow::Orchestrator.done_batch?(b) }.any?
+
+      candidates = Workflow::LocalExecutor.candidates(batches)
+      top_level_jobs = candidates.collect{|batch| batch[:top_level] }
+
+      raise NoWork, "No candidates and no running jobs #{Log.fingerprint batches}" if resources_used.empty? && top_level_jobs.empty?
+
+      candidates.each do |batch|
+        begin
+
+          job = batch[:top_level]
+
+          case
+          when (job.error? || job.aborted?)
+            begin
+              if job.recoverable_error?
+                if failed_jobs.include?(job)
+                  Log.warn "Failed twice #{job.path} with recoverable error"
+                  next
+                else
+                  failed_jobs << job
+                  job.clean
+                  raise TryAgain
+                end
+              else
+                Log.warn "Non-recoverable error in #{job.path}"
+                next
+              end
+            ensure
+              Log.warn "Releases resources from failed job: #{job.path}"
+              release_resources(job)
+            end
+          when job.done?
+            Log.debug "Orchestrator done #{job.path}"
+            release_resources(job)
+            clear_batch(batches, batch)
+            erase_job_dependencies(job, batches)
+          when job.running?
+            next
+
+          else
+            check_resources(batch) do
+              run_batch(batch)
+            end
+          end
+        rescue TryAgain
+          retry
+        end
+      end
+
+      batches.each do |batch|
+        job = batch[:top_level]
+        if job.done? || job.aborted? || job.error?
+          job.join if job.done?
+          clear_batch(batches, batch)
+          release_resources(job)
+          erase_job_dependencies(job, batches)
+        end
+      end
+
+      sleep timer
+    end
+
+    batches.each{|batch|
+      job = batch[:top_level]
+      begin
+        job.join
+      rescue
+        Log.warn "Job #{job.short_path} ended with exception #{$!.class.to_s}: #{$!.message}"
+      end
+    }
+
+    batches.each{|batch|
+      job = batch[:top_level]
+      erase_job_dependencies(job, batches) if job.done?
+    }
+  end
+
+  def process(rules, jobs = nil)
+    jobs, rules = rules, {} if jobs.nil?
+    jobs = [jobs] if Step === jobs
+
+    batches = Workflow::Orchestrator.job_batches(rules, jobs)
+    batches.each do |batch|
+      rules = IndiferentHash.setup batch[:rules]
+      rules.delete :erase if jobs.include?(batch[:top_level])
+      resources = Workflow::Orchestrator.normalize_resources_from_rules(rules)
+      resources = IndiferentHash.add_defaults resources, rules[:default_resources] if rules[:default_resources]
+      batch[:resources] = resources
+      batch[:rules] = rules
+    end
+
+    process_batches(batches)
+  end
+
+  def release_resources(job)
+    if resources_used[job]
+      Log.debug "Orchestrator releasing resouces from #{job.path}"
+      resources_used[job].each do |resource,value| 
+        next if resource == 'size'
+        resources_requested[resource] -= value.to_i
+      end
+      resources_used.delete job
+    end
+  end
+
+  def check_resources(batch)
+    resources = batch[:resources]
+    job = batch[:top_level]
+
+    limit_resources = resources.select do |resource,value|
+      value && available_resources[resource] && ((resources_requested[resource] || 0) + value) > available_resources[resource]
+    end.collect do |resource,v|
+      resource
+    end
+
+    if limit_resources.any?
+      Log.debug "Orchestrator waiting on #{job.path} due to #{limit_resources * ", "}"
+    else
+
+      resources_used[job] = resources
+      resources.each do |resource,value|
+        resources_requested[resource] ||= 0
+        resources_requested[resource] += value.to_i
+      end
+      Log.low "Orchestrator producing #{job.path} with resources #{resources}"
+
+      return yield
+    end
+  end
+
+  def run_batch(batch)
+    job, job_rules = batch.values_at :top_level, :rules
+
+    rules = batch[:rules] 
+    deploy = rules[:deploy] if rules
+    case deploy
+    when nil, 'local', :local, :serial, 'serial'
+      Scout::Config.with_config do
+        job_rules[:config_keys].split(/,\s*/).each do |config|
+          Scout::Config.process_config config
+        end if job_rules && job_rules[:config_keys]
+
+        log = job_rules[:log] if job_rules
+        log = Log.severity if log.nil?
+        Log.with_severity log do
+          job.fork
+        end
+      end
+    when 'batch', 'sched', 'slurm', 'pbs', 'lsf'
+      job.init_info
+      Workflow::Scheduler.process_batches([batch])
+      job.join
+    else
+      require 'scout/offsite'
+      if deploy.end_with?('-batch')
+        server = deploy.sub('-batch','')
+        OffsiteStep.setup(job, server: server, batch: true)
+      else
+        OffsiteStep.setup(job, server: deploy)
+      end
+
+      job.produce
+      job.join
+    end
+  end
+
+  def erase_job_dependencies(job, batches)
+    all_jobs = batches.collect{|b| b[:jobs] }.flatten
+    top_level_jobs = batches.collect{|b| b[:top_level] }
+
+    job.dependencies.each do |dep|
+      batch =  batches.select{|b| b[:jobs].include? dep}.first
+      next unless batch
+      rules = batch[:rules]
+      next unless rules[:erase].to_s == 'true'
+
+      dep_path = dep.path
+      parents = all_jobs.select do |parent|
+        parent.rec_dependencies.include?(dep)
+      end
+
+      next if parents.select{|parent| ! parent.done? }.any?
+
+      parents.each do |parent|
+        Log.high "Erasing #{dep.path} from #{parent.path}"
+        parent.archive_deps
+        parent.copy_linked_files_dir
+        parent.dependencies = parent.dependencies - [dep]
+      end
+
+      dep.clean
+    end
+  end
+
+  def clear_batch(batches, batch)
+    job = batch[:top_level]
+
+    parents = batches.select do |b|
+      b[:deps].include? batch
+    end
+
+    parents.each{|b| b[:deps].delete batch }
+  end
+
+  #{{{ HELPER
+
+  def self.purge_duplicates(batches)
+    seen = Set.new
+    batches.select do |batch|
+      path = batch[:top_level].path
+      if seen.include? path
+        false
+      else
+        seen << path
+        true
+      end
+    end
+  end
+
+  def self.sort_candidates(batches)
+    seen = Set.new
+    batches.sort_by do |batch|
+      - batch[:resources].values.compact.select{|e| Numeric === e }.inject(0.0){|acc,e| acc += e}
+    end
+  end
+
+  def self.candidates(batches)
+
+    leaf_nodes = batches.select{|b| b[:deps].empty? }
+
+    leaf_nodes.reject!{|b| Workflow::Orchestrator.done_batch?(b) }
+
+    leaf_nodes = purge_duplicates leaf_nodes
+    leaf_nodes = sort_candidates leaf_nodes
+
+    leaf_nodes
+  end
+end

data/lib/scout/workflow/deployment/orchestrator/batches.rb

@@ -0,0 +1,130 @@
+require_relative 'rules'
+require_relative 'workload'
+require_relative 'chains'
+
+class Workflow::Orchestrator
+
+  def self.chain_batches(rules, chains, workload)
+    chain_rules = parse_chains(rules)
+
+    batches = []
+    jobs = workload.keys
+    while job = jobs.pop
+      next if job.done?
+      matches = chains.select{|name,info| info[:jobs].include? job }
+      if matches.any?
+        name, info = matches.sort_by do |n, info|
+          num_jobs = info[:jobs].length
+          total_tasks = chain_rules[n][:tasks].values.flatten.uniq.length
+          num_jobs.to_f + 1.0/total_tasks
+        end.last
+        jobs = jobs - info[:jobs]
+        info[:chain] = name
+        batch = info
+      else
+        batch = {:jobs => [job], :top_level => job}
+      end
+
+      chains.delete_if{|n,info| batch[:jobs].include? info[:top_level] }
+
+      chains.each do |n,info|
+        info[:jobs] = info[:jobs] - batch[:jobs]
+      end
+
+      chains.delete_if{|n,info| info[:jobs].length < 2 }
+
+      batches << IndiferentHash.setup(batch)
+    end
+
+    batches
+  end
+
+  def self.add_batch_deps(batches)
+    batches.each do |batch|
+      jobs = batch[:jobs]
+      all_deps = jobs.collect{|j| job_dependencies(j) }.flatten.uniq - jobs
+
+      minimum = all_deps.dup
+      all_deps.each do |dep|
+        minimum -= job_dependencies(dep)
+      end
+
+      all_deps = minimum
+      deps = all_deps.collect do |d|
+        (batches - [batch]).select{|b| b[:jobs].collect(&:path).include? d.path }
+      end.flatten.uniq
+      batch[:deps] = deps
+    end
+
+    batches
+  end
+
+  def self.add_rules_and_consolidate(rules, batches)
+    chain_rules = parse_chains(rules)
+
+    batches.each do |batch|
+      job_rules_acc = batch[:jobs].inject(nil) do |acc, p|
+        job, deps = p
+        workflow = job.workflow
+        task_name = job.task_name
+        task_rules = task_specific_rules(rules, workflow, task_name)
+        acc = accumulate_rules(acc, task_rules.dup)
+      end
+
+      if chain = batch[:chain]
+        batch[:rules] = merge_rules(chain_rules[chain][:rules].dup, job_rules_acc)
+      else
+        batch[:rules] = job_rules_acc
+      end
+    end
+
+    begin
+      batches.each do |batch|
+        batch[:deps] = batch[:deps].collect do |dep|
+          dep[:target] || dep
+        end if batch[:deps]
+      end
+
+      batches.each do |batch|
+        next if batch[:top_level].overriden?
+        next unless batch[:rules] && batch[:rules][:skip]
+        batch[:rules].delete :skip
+        next if batch[:deps].nil?
+
+        if batch[:deps].any?
+          batch_dep_jobs = batch[:top_level].rec_dependencies
+          target = batch[:deps].select do |target|
+            batch_dep_jobs.include?(target[:top_level]) &&
+              (batch[:deps] - [target] - target[:deps]).empty?
+          end.first
+          next if target.nil?
+          target[:jobs] = batch[:jobs] + target[:jobs]
+          target[:deps] = (target[:deps] + batch[:deps]).uniq - [target]
+          target[:top_level] = batch[:top_level]
+          target[:rules] = accumulate_rules(target[:rules], batch[:rules])
+          batch[:target] = target
+        end
+        raise TryAgain
+      end
+    rescue TryAgain
+      retry
+    end
+
+    batches.delete_if{|b| b[:target] }
+
+    batches
+  end
+
+  def self.job_batches(rules, jobs)
+    jobs = [jobs] unless Array === jobs
+
+    workload = job_workload(jobs)
+    job_chains_map = jobs.inject([]){|acc,job| acc.concat(self.job_chains(rules, job)) }
+
+    batches = chain_batches(rules, job_chains_map, workload)
+    batches = add_batch_deps(batches)
+    batches = add_rules_and_consolidate(rules, batches)
+
+    batches
+  end
+end

data/lib/scout/workflow/deployment/orchestrator/chains.rb

@@ -0,0 +1,104 @@
+require_relative 'workload'
+
+class Workflow::Orchestrator
+  def self.check_chains(chains, job)
+    return [] if Symbol === job.overriden_task
+    matches = []
+    chains.each do |name, chain|
+      workflow = job.overriden_workflow || job.workflow
+      task_name = job.overriden_task || job.task_name
+      next unless chain[:tasks].include?(workflow.to_s)
+      next unless chain[:tasks][workflow.to_s].include?(task_name.to_s)
+      matches << name
+    end
+    matches
+  end
+
+  def self.parse_chains(rules)
+    rules = IndiferentHash.setup(rules || {})
+    chains = IndiferentHash.setup({})
+
+    # Rules may contain chains under workflows and/or top-level
+    rules.each do |workflow_name, wf_rules|
+      next unless wf_rules.is_a?(Hash)
+      next unless wf_rules["chains"]
+      wf_rules["chains"].each do |name, cr|
+        cr = IndiferentHash.setup(cr.dup)
+        chain_tasks = cr.delete(:tasks).to_s.split(/,\s*/)
+        wf = cr.delete(:workflow) if cr.include?(:workflow)
+
+        chain_tasks.each do |task|
+          chain_workflow, chain_task = task.split('#')
+          chain_task, chain_workflow = chain_workflow, wf if chain_task.nil? || chain_task.empty?
+
+          chains[name] ||= IndiferentHash.setup({:tasks => {}, :rules => cr })
+          chains[name][:tasks][chain_workflow] ||= []
+          chains[name][:tasks][chain_workflow] << chain_task
+        end
+      end
+    end
+
+    if rules["chains"]
+      rules["chains"].each do |name, cr|
+        cr = IndiferentHash.setup(cr.dup)
+        chain_tasks = cr.delete(:tasks).to_s.split(/,\s*/)
+        wf = cr.delete(:workflow)
+
+        chain_tasks.each do |task|
+          chain_workflow, chain_task = task.split('#')
+          chain_task, chain_workflow = chain_workflow, wf if chain_task.nil? || chain_task.empty?
+
+          chains[name] ||= IndiferentHash.setup({:tasks => {}, :rules => cr })
+          chains[name][:tasks][chain_workflow] ||= []
+          chains[name][:tasks][chain_workflow] << chain_task
+        end
+      end
+    end
+
+    chains
+  end
+
+  def self.job_chains(rules, job, computed = {})
+    key = Log.fingerprint([rules, job.path, job.object_id])
+    return computed[key] if computed.has_key?(key)
+
+    chains = parse_chains(rules)
+    matches = check_chains(chains, job)
+    dependencies = job_dependencies(job)
+
+    job_chains = []
+    new_job_chains = {}
+    dependencies.each do |dep|
+      dep_matches = check_chains(chains, dep)
+      common = matches & dep_matches
+
+      dep_chains = job_chains(rules, dep, computed)
+      found = []
+      dep_chains.each do |match, info|
+        if common.include?(match)
+          found << match
+          new_info = new_job_chains[match] ||= {}
+          new_info[:jobs] ||= []
+          new_info[:jobs].concat info[:jobs]
+          new_info[:top_level] = job
+        else
+          job_chains << [match, info]
+        end
+      end
+
+      (common - found).each do |match|
+          info = {}
+          info[:jobs] = [job, dep]
+          info[:top_level] = job
+          job_chains << [match, info]
+        end
+      end
+
+      new_job_chains.each do |match, info|
+        info[:jobs].prepend job
+        job_chains << [match, info]
+      end
+
+      computed[key] = job_chains
+    end
+end