scout-gear 10.9.0 → 10.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3900d5a0e546d494ae3224176db5feb92344dc1f6ba311461c74d02f5b137a9c
-  data.tar.gz: 35421139fea183944db794bad361bdfa9db78b086e13fab6367d6bbcfebc29e3
+  metadata.gz: 5f8aff3d8f7e3fc85a03340ec43ac1544b724a20de30e1607e1b07f6812099c7
+  data.tar.gz: 75685ce3a5868a2f21f71527fbbc69f46f758d279d393c2d1fc48c64ffa786f0
 SHA512:
-  metadata.gz: 679d05d7ae724825e18a2ce12862d08a3e75179f575a19ff960f4f379be6f2ce844326beca6e47d4bce3f585a4e637c71164b8642a2d61a85b2a54236f345153
-  data.tar.gz: c952339dfaa42d948a297a7f2c456d63bb4624d0ec71ee26499b79a2a25be8f32ec8c2b5ea2bd9cc17a664c50edb8f6cf22dec458ea103eb6e82614774973bdc
+  metadata.gz: 2e20ac4a92bfed76ea6967cc3441ebf0bd33f72203592dec178f88edb094e30be01f2fc84e749f002be5987baffd2e10a7a0b9653bc0ef14c04da31633306f89
+  data.tar.gz: 12bc3dbcad9f8ba28c6472fadc6f525813a02f4a631010422f01984753f39e429e5ea8cf958bc2e160ccd11d6bc8a798251b32043b1b63fc4cc9960d2ec9281c

data/.vimproject

@@ -2,6 +2,10 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
  Rakefile
  README.md
  chats=chats filter="*"{
+
+  deploy
+
+
   debug
   document.rb
   pipes
@@ -46,10 +50,31 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      inputs.rb
      info.rb
     }
+
     deployment.rb
     deployment=deployment{
+     orchestrator=orchestrator{
+      rules.rb
+      workload.rb
+      chains.rb
+      batches.rb
+     }
+
+     local.rb
+
+     scheduler.rb
+
+     scheduler=scheduler{
+      job.rb
+      lfs.rb
+      slurm.rb
+      pbs.rb
+     }
+
      queue.rb
+
      trace.rb
+
      orchestrator.rb
     }
     entity.rb

data/VERSION

@@ -1 +1 @@
-10.9.0
+10.11.0

data/bin/scout

@@ -90,7 +90,7 @@ end
 if options[:log]
   Log.severity = options.delete(:log).to_i
   require 'scout-gear'
-else
+elsif Log.severity.nil?
   require 'scout-gear'
 
   global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
@@ -100,8 +100,11 @@ else
     global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
     Log.severity = global_severity.to_i if global_severity
   end
+else
+  require 'scout-gear'
 end
 
+
 if config_keys = options.delete(:config_keys)
   config_keys.split(",").each do |config|
     config = config.strip

data/lib/scout/knowledge_base/registry.rb

@@ -175,14 +175,13 @@ class KnowledgeBase
 
           database = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
                        Log.low "Re-opening database #{ name } from #{ Log.fingerprint persist_path }. #{options}"
-                       Association.database(file, **options.merge(persist_options: persist_options))
+                       Association.database(file, **options.merge(persist_options: persist_options).except(:undirected))
                      else
                        options = IndiferentHash.add_defaults options, registered_options if registered_options
-                       undirected = IndiferentHash.process_options options, :undirected
                        raise "Repo #{ name } not found and not registered" if file.nil?
                        Log.medium "Opening database #{ name } from #{ Log.fingerprint file }. #{options}"
                        file = file.call if Proc === file
-                       Association.database(file, **options.merge(persist_options: persist_options))
+                       Association.database(file, **options.merge(persist_options: persist_options).except(:undirected))
                      end
 
           database.namespace = self.namespace if self.namespace

data/lib/scout/work_queue/worker.rb

@@ -1,5 +1,8 @@
 class WorkQueue
   class Worker
+    EXIT_STATUS=246
+    SIGNAL='ABRT'
+
     attr_accessor :pid, :ignore_ouput, :queue_id
     def initialize(ignore_ouput = false)
       @ignore_output = ignore_ouput
@@ -15,7 +18,10 @@ class WorkQueue
 
     def run
       @pid = Process.fork do
-        Signal.trap("INT") do
+        Signal.trap(SIGNAL) do
+          Kernel.exit! EXIT_STATUS
+        end
+        Signal.trap('INT') do
           Kernel.exit! -1
         end
         Log.low "Worker start #{worker_id}"
@@ -46,7 +52,7 @@ class WorkQueue
           begin
             output.write WorkerException.new($!, Process.pid)
           ensure
-            exit -1
+            exit EXIT_STATUS
           end
         end
         exit 0
@@ -56,7 +62,7 @@ class WorkQueue
     def abort
       begin
         Log.medium "Aborting worker #{worker_id}"
-        Process.kill "INT", @pid
+        Process.kill SIGNAL, @pid
       rescue Errno::ECHILD 
       rescue Errno::ESRCH
       end

data/lib/scout/work_queue.rb

@@ -71,13 +71,15 @@ class WorkQueue
         Thread.current.report_on_exception = false
         Thread.current["name"] = "Output reader #{queue_id}"
         @done_workers ||= []
-        while true
-          obj = @output.read
+        #while true
+        #  obj = @output.read
+        while obj = @output.read
           if DoneProcessing === obj
 
             done = @worker_mutex.synchronize do
               Log.low "Worker #{obj.pid} from #{queue_id} done"
               @done_workers << obj.pid
+              #@closed && (@workers.empty? || @workers.length == @removed_workers.length + @done_workers.length)
               @closed && @done_workers.length == @removed_workers.length + @workers.length
             end
 
@@ -113,14 +115,29 @@ class WorkQueue
         break if @worker_mutex.synchronize{ @workers.empty? }
         threads = @workers.collect do |w|
           t = Thread.new do
+            Thread.report_on_exception = false
             Thread.current["name"] = "Worker waiter #{queue_id} worker #{w.pid}"
             pid, status = Process.wait2 w.pid
             remove_worker(pid) if pid
+            #@output.write WorkerException.new(Exception.new("Worker ended with status #{status.exitstatus}"), pid) unless status.success?
+            raise Exception.new("Worker #{pid} ended with status #{status.exitstatus}") unless (status.success? || status.exitstatus == WorkQueue::Worker::EXIT_STATUS)
           end
           Thread.pass until t["name"]
           t
         end
-        threads.each do |t| t.join end
+        exceptions = []
+        threads.each do |t| 
+          begin
+            t.join 
+          rescue
+            exceptions << $!
+          end
+        end
+
+        raise exceptions.first if exceptions.any?
+        if @workers.empty? && ! @closed
+          @output.write DoneProcessing.new
+        end
       end
     end
 

data/lib/scout/workflow/definition.rb

@@ -227,4 +227,15 @@ module Workflow
   alias export_asynchronous export
   alias export_exec export
   alias export_stream export
+
+  def include_workflow(workflow)
+    workflow.documentation
+    self.asynchronous_exports += workflow.asynchronous_exports
+    self.synchronous_exports += workflow.synchronous_exports
+    self.exec_exports += workflow.exec_exports
+    self.stream_exports += workflow.stream_exports
+    self.tasks.merge! workflow.tasks
+    self.tasks.each{|_,t| t.workflow = workflow }
+    self.helpers.merge! workflow.helpers
+  end
 end

data/lib/scout/workflow/deployment/local.rb

@@ -0,0 +1,288 @@
+require_relative 'orchestrator/batches'
+class Workflow::LocalExecutor
+  class NoWork < Exception; end
+
+  def self.process(*args)
+    self.new.process(*args)
+  end
+
+  def self.produce(jobs, rules = {}, produce_cpus: Etc.nprocessors, produce_timer: 1)
+    jobs = [jobs] unless Array === jobs
+    orchestrator = self.new produce_timer.to_f, cpus: produce_cpus.to_i
+    begin
+      orchestrator.process(rules, jobs)
+    rescue self::NoWork
+    end
+  end
+
+  def self.produce_dependencies(jobs, tasks, rules = {}, produce_cpus: Etc.nprocessors, produce_timer: 1)
+    jobs = [jobs] unless Array === jobs
+    tasks = tasks.collect{|task| (String === task) ? task.to_sym : task }
+
+    produce_list = []
+    jobs.each do |job|
+      next if job.done? || job.running?
+      job.rec_dependencies.each do |dep|
+        task_name = dep.task_name.to_sym
+        task_name = task_name.to_sym if String === task_name
+        produce_list << dep if tasks.include?(task_name) ||
+          tasks.include?(job.task_name.to_s) ||
+          tasks.include?(job.full_task_name)
+      end
+    end
+
+    produce(produce_list, rules, produce_cpus: produce_cpus, produce_timer: produce_timer)
+  end
+
+  attr_accessor :available_resources, :resources_requested, :resources_used, :timer
+
+  def initialize(timer = 5, available_resources = nil)
+    available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
+    @timer               = timer
+    @available_resources = IndiferentHash.setup(available_resources)
+    @resources_requested = IndiferentHash.setup({})
+    @resources_used      = IndiferentHash.setup({})
+  end
+
+  def process_batches(batches)
+    failed_jobs = []
+
+    while batches.reject{|b| Workflow::Orchestrator.done_batch?(b) }.any?
+
+      candidates = Workflow::LocalExecutor.candidates(batches)
+      top_level_jobs = candidates.collect{|batch| batch[:top_level] }
+
+      raise NoWork, "No candidates and no running jobs #{Log.fingerprint batches}" if resources_used.empty? && top_level_jobs.empty?
+
+      candidates.each do |batch|
+        begin
+
+          job = batch[:top_level]
+
+          case
+          when (job.error? || job.aborted?)
+            begin
+              if job.recoverable_error?
+                if failed_jobs.include?(job)
+                  Log.warn "Failed twice #{job.path} with recoverable error"
+                  next
+                else
+                  failed_jobs << job
+                  job.clean
+                  raise TryAgain
+                end
+              else
+                Log.warn "Non-recoverable error in #{job.path}"
+                next
+              end
+            ensure
+              Log.warn "Releases resources from failed job: #{job.path}"
+              release_resources(job)
+            end
+          when job.done?
+            Log.debug "Orchestrator done #{job.path}"
+            release_resources(job)
+            clear_batch(batches, batch)
+            erase_job_dependencies(job, batches)
+          when job.running?
+            next
+
+          else
+            check_resources(batch) do
+              run_batch(batch)
+            end
+          end
+        rescue TryAgain
+          retry
+        end
+      end
+
+      batches.each do |batch|
+        job = batch[:top_level]
+        if job.done? || job.aborted? || job.error?
+          job.join if job.done?
+          clear_batch(batches, batch)
+          release_resources(job)
+          erase_job_dependencies(job, batches)
+        end
+      end
+
+      sleep timer
+    end
+
+    batches.each{|batch|
+      job = batch[:top_level]
+      begin
+        job.join
+      rescue
+        Log.warn "Job #{job.short_path} ended with exception #{$!.class.to_s}: #{$!.message}"
+      end
+    }
+
+    batches.each{|batch|
+      job = batch[:top_level]
+      erase_job_dependencies(job, batches) if job.done?
+    }
+  end
+
+  def process(rules, jobs = nil)
+    jobs, rules = rules, {} if jobs.nil?
+    jobs = [jobs] if Step === jobs
+
+    batches = Workflow::Orchestrator.job_batches(rules, jobs)
+    batches.each do |batch|
+      rules = IndiferentHash.setup batch[:rules]
+      rules.delete :erase if jobs.include?(batch[:top_level])
+      resources = Workflow::Orchestrator.normalize_resources_from_rules(rules)
+      resources = IndiferentHash.add_defaults resources, rules[:default_resources] if rules[:default_resources]
+      batch[:resources] = resources
+      batch[:rules] = rules
+    end
+
+    process_batches(batches)
+  end
+
+  def release_resources(job)
+    if resources_used[job]
+      Log.debug "Orchestrator releasing resouces from #{job.path}"
+      resources_used[job].each do |resource,value| 
+        next if resource == 'size'
+        resources_requested[resource] -= value.to_i
+      end
+      resources_used.delete job
+    end
+  end
+
+  def check_resources(batch)
+    resources = batch[:resources]
+    job = batch[:top_level]
+
+    limit_resources = resources.select do |resource,value|
+      value && available_resources[resource] && ((resources_requested[resource] || 0) + value) > available_resources[resource]
+    end.collect do |resource,v|
+      resource
+    end
+
+    if limit_resources.any?
+      Log.debug "Orchestrator waiting on #{job.path} due to #{limit_resources * ", "}"
+    else
+
+      resources_used[job] = resources
+      resources.each do |resource,value|
+        resources_requested[resource] ||= 0
+        resources_requested[resource] += value.to_i
+      end
+      Log.low "Orchestrator producing #{job.path} with resources #{resources}"
+
+      return yield
+    end
+  end
+
+  def run_batch(batch)
+    job, job_rules = batch.values_at :top_level, :rules
+
+    rules = batch[:rules] 
+    deploy = rules[:deploy] if rules
+    case deploy
+    when nil, 'local', :local, :serial, 'serial'
+      Scout::Config.with_config do
+        job_rules[:config_keys].split(/,\s*/).each do |config|
+          Scout::Config.process_config config
+        end if job_rules && job_rules[:config_keys]
+
+        log = job_rules[:log] if job_rules
+        log = Log.severity if log.nil?
+        Log.with_severity log do
+          job.fork
+        end
+      end
+    when 'batch', 'sched', 'slurm', 'pbs', 'lsf'
+      job.init_info
+      Workflow::Scheduler.process_batches([batch])
+      job.join
+    else
+      require 'scout/offsite'
+      if deploy.end_with?('-batch')
+        server = deploy.sub('-batch','')
+        OffsiteStep.setup(job, server: server, batch: true)
+      else
+        OffsiteStep.setup(job, server: deploy)
+      end
+
+      job.produce
+      job.join
+    end
+  end
+
+  def erase_job_dependencies(job, batches)
+    all_jobs = batches.collect{|b| b[:jobs] }.flatten
+    top_level_jobs = batches.collect{|b| b[:top_level] }
+
+    job.dependencies.each do |dep|
+      batch =  batches.select{|b| b[:jobs].include? dep}.first
+      next unless batch
+      rules = batch[:rules]
+      next unless rules[:erase].to_s == 'true'
+
+      dep_path = dep.path
+      parents = all_jobs.select do |parent|
+        parent.rec_dependencies.include?(dep)
+      end
+
+      next if parents.select{|parent| ! parent.done? }.any?
+
+      parents.each do |parent|
+        Log.high "Erasing #{dep.path} from #{parent.path}"
+        parent.archive_deps
+        parent.copy_linked_files_dir
+        parent.dependencies = parent.dependencies - [dep]
+      end
+
+      dep.clean
+    end
+  end
+
+  def clear_batch(batches, batch)
+    job = batch[:top_level]
+
+    parents = batches.select do |b|
+      b[:deps].include? batch
+    end
+
+    parents.each{|b| b[:deps].delete batch }
+  end
+
+  #{{{ HELPER
+
+  def self.purge_duplicates(batches)
+    seen = Set.new
+    batches.select do |batch|
+      path = batch[:top_level].path
+      if seen.include? path
+        false
+      else
+        seen << path
+        true
+      end
+    end
+  end
+
+  def self.sort_candidates(batches)
+    seen = Set.new
+    batches.sort_by do |batch|
+      - batch[:resources].values.compact.select{|e| Numeric === e }.inject(0.0){|acc,e| acc += e}
+    end
+  end
+
+  def self.candidates(batches)
+
+    leaf_nodes = batches.select{|b| b[:deps].empty? }
+
+    leaf_nodes.reject!{|b| Workflow::Orchestrator.done_batch?(b) }
+
+    leaf_nodes = purge_duplicates leaf_nodes
+    leaf_nodes = sort_candidates leaf_nodes
+
+    leaf_nodes
+  end
+end

data/lib/scout/workflow/deployment/orchestrator/batches.rb

@@ -0,0 +1,130 @@
+require_relative 'rules'
+require_relative 'workload'
+require_relative 'chains'
+
+class Workflow::Orchestrator
+
+  def self.chain_batches(rules, chains, workload)
+    chain_rules = parse_chains(rules)
+
+    batches = []
+    jobs = workload.keys
+    while job = jobs.pop
+      next if job.done?
+      matches = chains.select{|name,info| info[:jobs].include? job }
+      if matches.any?
+        name, info = matches.sort_by do |n, info|
+          num_jobs = info[:jobs].length
+          total_tasks = chain_rules[n][:tasks].values.flatten.uniq.length
+          num_jobs.to_f + 1.0/total_tasks
+        end.last
+        jobs = jobs - info[:jobs]
+        info[:chain] = name
+        batch = info
+      else
+        batch = {:jobs => [job], :top_level => job}
+      end
+
+      chains.delete_if{|n,info| batch[:jobs].include? info[:top_level] }
+
+      chains.each do |n,info|
+        info[:jobs] = info[:jobs] - batch[:jobs]
+      end
+
+      chains.delete_if{|n,info| info[:jobs].length < 2 }
+
+      batches << IndiferentHash.setup(batch)
+    end
+
+    batches
+  end
+
+  def self.add_batch_deps(batches)
+    batches.each do |batch|
+      jobs = batch[:jobs]
+      all_deps = jobs.collect{|j| job_dependencies(j) }.flatten.uniq - jobs
+
+      minimum = all_deps.dup
+      all_deps.each do |dep|
+        minimum -= job_dependencies(dep)
+      end
+
+      all_deps = minimum
+      deps = all_deps.collect do |d|
+        (batches - [batch]).select{|b| b[:jobs].collect(&:path).include? d.path }
+      end.flatten.uniq
+      batch[:deps] = deps
+    end
+
+    batches
+  end
+
+  def self.add_rules_and_consolidate(rules, batches)
+    chain_rules = parse_chains(rules)
+
+    batches.each do |batch|
+      job_rules_acc = batch[:jobs].inject(nil) do |acc, p|
+        job, deps = p
+        workflow = job.workflow
+        task_name = job.task_name
+        task_rules = task_specific_rules(rules, workflow, task_name)
+        acc = accumulate_rules(acc, task_rules.dup)
+      end
+
+      if chain = batch[:chain]
+        batch[:rules] = merge_rules(chain_rules[chain][:rules].dup, job_rules_acc)
+      else
+        batch[:rules] = job_rules_acc
+      end
+    end
+
+    begin
+      batches.each do |batch|
+        batch[:deps] = batch[:deps].collect do |dep|
+          dep[:target] || dep
+        end if batch[:deps]
+      end
+
+      batches.each do |batch|
+        next if batch[:top_level].overriden?
+        next unless batch[:rules] && batch[:rules][:skip]
+        batch[:rules].delete :skip
+        next if batch[:deps].nil?
+
+        if batch[:deps].any?
+          batch_dep_jobs = batch[:top_level].rec_dependencies
+          target = batch[:deps].select do |target|
+            batch_dep_jobs.include?(target[:top_level]) &&
+              (batch[:deps] - [target] - target[:deps]).empty?
+          end.first
+          next if target.nil?
+          target[:jobs] = batch[:jobs] + target[:jobs]
+          target[:deps] = (target[:deps] + batch[:deps]).uniq - [target]
+          target[:top_level] = batch[:top_level]
+          target[:rules] = accumulate_rules(target[:rules], batch[:rules])
+          batch[:target] = target
+        end
+        raise TryAgain
+      end
+    rescue TryAgain
+      retry
+    end
+
+    batches.delete_if{|b| b[:target] }
+
+    batches
+  end
+
+  def self.job_batches(rules, jobs)
+    jobs = [jobs] unless Array === jobs
+
+    workload = job_workload(jobs)
+    job_chains_map = jobs.inject([]){|acc,job| acc.concat(self.job_chains(rules, job)) }
+
+    batches = chain_batches(rules, job_chains_map, workload)
+    batches = add_batch_deps(batches)
+    batches = add_rules_and_consolidate(rules, batches)
+
+    batches
+  end
+end