scout-gear 10.11.6 → 10.11.8

data/lib/scout/tsv/traverse.rb

@@ -60,9 +60,9 @@ module TSV
                 key = values.delete_at(key_pos)
               end
             end
-          else 
+          else
             orig_key = key
-            key = @type == :flat ? values : values[key_pos] if key_pos != :key 
+            key = @type == :flat ? values : values[key_pos] if key_pos != :key
 
             values = values.values_at(*positions)
             if key_index
@@ -85,7 +85,7 @@ module TSV
 
           values = TSV.cast_value(values, cast) if cast
 
-          if Array === key 
+          if Array === key
             key = key.uniq if uniq
             if @type == :double && one2one
               if one2one == :strict
@@ -159,7 +159,7 @@ module TSV
         end
       end
     end
-    
+
 
     [key_name, field_names]
   end

data/lib/scout/tsv/util/filter.rb

@@ -27,7 +27,7 @@ module Filtered
       @value = value
       @unsaved = []
 
-      case 
+      case
       when Hash === persistence
         @persistence = persistence
       when String === persistence
@@ -57,7 +57,7 @@ module Filtered
           self
         end.class_eval <<-EOC
           def match_entry(key, entry)
-            value = entry[@fieldnum] 
+            value = entry[@fieldnum]
             value == @value or (Array === value and value.include? @value)
           end
         EOC
@@ -67,7 +67,7 @@ module Filtered
     end
 
     def key
-      case 
+      case
       when String === value
         value
       else
@@ -162,7 +162,7 @@ module Filtered
     if not base.respond_to? :unfiltered_set
       class << base
         attr_accessor :filter_dir, :filters
-        
+
         alias unfiltered_set []=
         alias []= filtered_set
 
@@ -200,7 +200,7 @@ module Filtered
     if filters.empty?
       self.send(:unfiltered_set, key, value, clean)
     else
-      filters.each do |filter| 
+      filters.each do |filter|
         filter.add key if filter.match_entry key, value
       end
       self.send(:unfiltered_set, key, value, clean)
@@ -215,10 +215,10 @@ module Filtered
     end
   end
 
-  def filtered_values 
-    if filters.empty?  
-      self.send(:unfiltered_values) 
-    else 
+  def filtered_values
+    if filters.empty?
+      self.send(:unfiltered_values)
+    else
       ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
       self.send :values_at, *ids
     end

data/lib/scout/tsv/util/process.rb

@@ -13,7 +13,7 @@ module TSV
         field_values = values[field_pos]
       end
 
-      new_values = case 
+      new_values = case
                    when block.arity == 1
                      yield(field_values)
                    when block.arity == 2
@@ -54,7 +54,7 @@ module TSV
         values = new_values
       when (values.nil? and (fields.nil? or fields.empty?))
         values = [new_values]
-      when values.nil?  
+      when values.nil?
         values = [nil] * fields.length + [new_values]
       when Array === values
         values += [new_values]

data/lib/scout/tsv/util/reorder.rb

@@ -1,7 +1,7 @@
 require 'matrix'
 
 module TSV
-  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, data: nil, unnamed: true, **kwargs) 
+  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, data: nil, unnamed: true, **kwargs)
     res = data || self.annotate({})
     res.type = kwargs[:type] if kwargs.include?(:type)
     kwargs[:one2one] = one2one
@@ -61,7 +61,7 @@ module TSV
     new = self.annotate({})
     TSV.setup(new, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
 
-    m = Matrix.rows values 
+    m = Matrix.rows values
     new_rows = m.transpose.to_a
 
     fields.zip(new_rows) do |key,row|

data/lib/scout/tsv/util/select.rb

@@ -41,7 +41,7 @@ module TSV
       if block_given?
         field = method
         field = fields.index?(field) if fields && String === field
-        case 
+        case
         when block.arity == 1
           if (method == key_field or method == :key)
             yield(key)
@@ -69,7 +69,7 @@ module TSV
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 
     self.annotate(new)
-    
+
     case
     when (method.nil? and block_given?)
       through do |key, values|
@@ -101,7 +101,7 @@ module TSV
       end
     when ((String === method) || (Symbol === method))
       if block_given?
-        case 
+        case
         when block.arity == 1
           with_unnamed do
             case

data/lib/scout/tsv/util/unzip.rb

@@ -20,7 +20,7 @@ module TSV
              else
                target
              end
-               
+
     target.fields = new_fields
     target.key_field = new_key_field
     target.type = type
@@ -57,7 +57,7 @@ module TSV
             [new_key, new_values]
           end
         end
-        
+
         MultipleResult.setup(res)
       else
         field_value = v[field_pos]

data/lib/scout/tsv/util.rb

@@ -129,7 +129,7 @@ module TSV
 
     filename = @filename
     filename = "No filename" if filename.nil? || String === filename && filename.empty?
-    filename.find if Path === filename 
+    filename.find if Path === filename
     filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
 
     with_unnamed do

data/lib/scout/tsv.rb

@@ -68,7 +68,7 @@ module TSV
   end
 
   def self.str_setup(option_str, obj)
-    options = TSV.str2options(option_str) 
+    options = TSV.str2options(option_str)
     setup(obj, **options)
   end
 
@@ -87,7 +87,7 @@ module TSV
 
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
 
-    source_name, options = 
+    source_name, options =
       case file
       when StringIO
         [file.inspect, options]

data/lib/scout/work_queue/socket.rb

@@ -57,9 +57,9 @@ class WorkQueue
       end
 
       write_length = str.length
-      wrote = stream.write(str) 
+      wrote = stream.write(str)
       while wrote < write_length
-        wrote += stream.write(str[wrote..-1]) 
+        wrote += stream.write(str[wrote..-1])
       end
     end
 

data/lib/scout/work_queue/worker.rb

@@ -41,10 +41,10 @@ class WorkQueue
           while obj = input.read
             if DoneProcessing === obj
               output.write DoneProcessing.new
-              raise obj 
+              raise obj
             end
             res = block.call obj
-            output.write res unless ignore_ouput || res == :ignore 
+            output.write res unless ignore_ouput || res == :ignore
           end
         rescue DoneProcessing
         rescue Interrupt
@@ -64,7 +64,7 @@ class WorkQueue
       begin
         Log.medium "Aborting worker #{worker_id}"
         Process.kill SIGNAL, @pid
-      rescue Errno::ECHILD 
+      rescue Errno::ECHILD
       rescue Errno::ESRCH
       end
     end
@@ -77,7 +77,7 @@ class WorkQueue
     def self.join(workers)
       workers = [workers] unless Array === workers
       begin
-        while pid = Process.wait 
+        while pid = Process.wait
           status = $?
             worker = workers.select{|w| w.pid == pid }.first
         end

data/lib/scout/work_queue.rb

@@ -62,7 +62,7 @@ class WorkQueue
   end
 
   def process(&callback)
-    @workers.each do |w| 
+    @workers.each do |w|
       w.process @input, @output, &@worker_proc
     end
 
@@ -126,9 +126,9 @@ class WorkQueue
           t
         end
         exceptions = []
-        threads.each do |t| 
+        threads.each do |t|
           begin
-            t.join 
+            t.join
           rescue Exception
             exceptions << $!
           end
@@ -155,10 +155,10 @@ class WorkQueue
     @aborted = true
     Log.low "Aborting #{@workers.length} workers in queue #{queue_id}"
     @worker_mutex.synchronize do
-      @workers.each do |w| 
+      @workers.each do |w|
         ScoutSemaphore.post_semaphore(@output.write_sem)
         ScoutSemaphore.post_semaphore(@input.read_sem)
-        w.abort 
+        w.abort
       end
     end
   end

data/lib/scout/workflow/definition.rb

@@ -123,7 +123,7 @@ module Workflow
     block = lambda &self.method(name) if block.nil?
     begin
       @annotate_next_task ||= {}
-      @annotate_next_task[:extension] ||=  
+      @annotate_next_task[:extension] ||=
         case type
         when :tsv
           "tsv"
@@ -144,24 +144,24 @@ module Workflow
     end
   end
 
-  FORGET_TASK_ALIAS = begin 
+  FORGET_TASK_ALIAS = begin
                         %w(SCOUT_FORGET_TASK_ALIAS SCOUT_FORGET_DEP_TASKS RBBT_FORGET_DEP_TASKS).select do |var|
                           ENV[var] == 'true'
                         end.any?
                       end
-  REMOVE_TASK_ALIAS = begin 
+  REMOVE_TASK_ALIAS = begin
                         remove = %w(SCOUT_REMOVE_TASK_ALIAS SCOUT_REMOVE_DEP_TASKS RBBT_REMOVE_DEP_TASKS).select do |var|
                           ENV.include?(var) && ENV[var] != 'false'
                         end.first
                         remove.nil? ? false : remove
                       end
   def task_alias(name, workflow, oname, *rest, &block)
-    dep(workflow, oname, *rest, &block) 
+    dep(workflow, oname, *rest, &block)
     extension :dep_task unless @extension
     task_proc = workflow.tasks[oname] if workflow.tasks
     if task_proc
       returns task_proc.returns if @returns.nil?
-      type = task_proc.type 
+      type = task_proc.type
     end
     task name => type do
       raise ScoutException, "dep_task does not have any dependencies" if dependencies.empty?
@@ -187,25 +187,27 @@ module Workflow
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         Open.link_dir dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
 
-        if dep.overriden? 
+        if dep.overriden? || dep.overrider?
           Open.link dep.path, self.tmp_path
         else
           Open.ln_h dep.path, self.tmp_path
 
-          case remove.to_s
-          when 'true'
-            dep.clean
-          when 'recursive'
-            (dep.dependencies.to_a + dep.rec_dependencies.to_a).uniq.each do |d|
-              next if d.overriden
-              d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
+          if Open.exists?(dep.info_file)
+            case remove.to_s
+            when 'true'
+              dep.clean
+            when 'recursive'
+              (dep.dependencies.to_a + dep.rec_dependencies.to_a).uniq.each do |d|
+                next if d.overriden
+                d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
+              end
+              dep.clean unless Scout::Config.get(:remove_dep, "task:#{dep.task_signature}", "task:#{dep.task_name}", "workflow:#{dep.workflow.name}", :default => true).to_s == 'false'
             end
-            dep.clean unless Scout::Config.get(:remove_dep, "task:#{dep.task_signature}", "task:#{dep.task_name}", "workflow:#{dep.workflow.name}", :default => true).to_s == 'false'
-          end 
+          end
         end
       else
         if Open.exists?(dep.files_dir)
-          Open.rm_rf self.files_dir 
+          Open.rm_rf self.files_dir
           Open.link dep.files_dir, self.files_dir
         end
         if defined?(RemoteStep) && RemoteStep === dep

data/lib/scout/workflow/deployment/local.rb

@@ -42,86 +42,95 @@ class Workflow::LocalExecutor
     @available_resources = IndiferentHash.setup(available_resources)
     @resources_requested = IndiferentHash.setup({})
     @resources_used      = IndiferentHash.setup({})
+    Log.info "LocalExecutor initiated #{Log.fingerprint available_resources}"
   end
 
-  def process_batches(batches)
+  def process_batches(batches, bar: true)
     retry_jobs = []
     failed_jobs = []
 
-    while batches.reject{|b| Workflow::Orchestrator.done_batch?(b) }.any?
+    bar = {desc: "Processing batches"} if TrueClass === bar
+    bar = {bar: bar} if Log::ProgressBar === bar
+    Log::ProgressBar.with_bar batches.length, bar do |bar|
+      bar.init if bar
 
-      candidates = Workflow::LocalExecutor.candidates(batches)
-      top_level_jobs = candidates.collect{|batch| batch[:top_level] }
+      while (missing_batches = batches.reject{|b| Workflow::Orchestrator.done_batch?(b) }).any?
 
-      raise NoWork, "No candidates and no running jobs #{Log.fingerprint batches}" if resources_used.empty? && top_level_jobs.empty?
+        bar.pos batches.select{|b| Workflow::Orchestrator.done_batch?(b) }.length if bar
 
-      if candidates.reject{|batch| failed_jobs.include? batch[:top_level] }.empty? && resources_used.empty? && top_level_jobs.empty?
-        exception = failed_jobs.collect(&:get_exception).compact.first
-        if exception
-          Log.warn 'Some work failed'
-          raise exception 
-        else
-          raise 'Some work failed'
-        end
-      end
+        candidates = Workflow::LocalExecutor.candidates(batches)
+        top_level_jobs = candidates.collect{|batch| batch[:top_level] }
 
-      candidates.each do |batch|
-        begin
+        raise NoWork, "No candidates and no running jobs #{Log.fingerprint batches}" if resources_used.empty? && top_level_jobs.empty?
 
-          job = batch[:top_level]
+        if candidates.reject{|batch| failed_jobs.include? batch[:top_level] }.empty? && resources_used.empty? && top_level_jobs.empty?
+          exception = failed_jobs.collect(&:get_exception).compact.first
+          if exception
+            Log.warn 'Some work failed'
+            raise exception
+          else
+            raise 'Some work failed'
+          end
+        end
 
-          case
-          when (job.error? || job.aborted?)
-            begin
-              if job.recoverable_error?
-                if retry_jobs.include?(job)
-                  Log.warn "Failed twice #{job.path} with recoverable error"
-                  retry_jobs.delete job
+        candidates.each do |batch|
+          begin
+
+            job = batch[:top_level]
+
+            case
+            when (job.error? || job.aborted?)
+              begin
+                if job.recoverable_error?
+                  if retry_jobs.include?(job)
+                    Log.warn "Failed twice #{job.path} with recoverable error"
+                    retry_jobs.delete job
+                    failed_jobs << job
+                    next
+                  else
+                    retry_jobs << job
+                    job.clean
+                    raise TryAgain
+                  end
+                else
                   failed_jobs << job
+                  Log.warn "Non-recoverable error in #{job.path}"
                   next
-                else
-                  retry_jobs << job
-                  job.clean
-                  raise TryAgain
                 end
-              else
-                failed_jobs << job
-                Log.warn "Non-recoverable error in #{job.path}"
-                next
+              ensure
+                Log.warn "Releases resources from failed job: #{job.path}"
+                release_resources(job)
               end
-            ensure
-              Log.warn "Releases resources from failed job: #{job.path}"
+            when job.done?
+              Log.debug "Orchestrator done #{job.path}"
               release_resources(job)
+              clear_batch(batches, batch)
+              erase_job_dependencies(job, batches)
+            when job.running?
+              next
+
+            else
+              check_resources(batch) do
+                run_batch(batch)
+              end
             end
-          when job.done?
-            Log.debug "Orchestrator done #{job.path}"
-            release_resources(job)
+          rescue TryAgain
+            retry
+          end
+        end
+
+        batches.each do |batch|
+          job = batch[:top_level]
+          if job.done? || job.aborted? || job.error?
+            job.join if job.done?
             clear_batch(batches, batch)
+            release_resources(job)
             erase_job_dependencies(job, batches)
-          when job.running?
-            next
-
-          else
-            check_resources(batch) do
-              run_batch(batch)
-            end
           end
-        rescue TryAgain
-          retry
         end
-      end
 
-      batches.each do |batch|
-        job = batch[:top_level]
-        if job.done? || job.aborted? || job.error?
-          job.join if job.done?
-          clear_batch(batches, batch)
-          release_resources(job)
-          erase_job_dependencies(job, batches)
-        end
+        sleep timer
       end
-
-      sleep timer
     end
 
     batches.each{|batch|
@@ -141,9 +150,19 @@ class Workflow::LocalExecutor
 
   def process(rules, jobs = nil)
     jobs, rules = rules, {} if jobs.nil?
-    jobs = [jobs] if Step === jobs
+
+    if Step === jobs
+      jobs = [jobs]
+    end
 
     batches = Workflow::Orchestrator.job_batches(rules, jobs)
+
+    if jobs.length == 1
+      bar = jobs.first.progress_bar("Processing batches for #{jobs.first.short_path}", max: batches.length)
+    else
+      bar = true
+    end
+
     batches.each do |batch|
       rules = IndiferentHash.setup batch[:rules]
       rules.delete :erase if jobs.include?(batch[:top_level])
@@ -153,13 +172,13 @@ class Workflow::LocalExecutor
       batch[:rules] = rules
     end
 
-    process_batches(batches)
+    process_batches(batches, bar: bar)
   end
 
   def release_resources(job)
     if resources_used[job]
       Log.debug "Orchestrator releasing resouces from #{job.path}"
-      resources_used[job].each do |resource,value| 
+      resources_used[job].each do |resource,value|
         next if resource == 'size'
         resources_requested[resource] -= value.to_i
       end
@@ -195,8 +214,9 @@ class Workflow::LocalExecutor
   def run_batch(batch)
     job, job_rules = batch.values_at :top_level, :rules
 
-    rules = batch[:rules] 
+    rules = batch[:rules]
     deploy = rules[:deploy] if rules
+    Log.debug "Processing #{deploy} #{job.short_path} #{Log.fingerprint job_rules}"
     case deploy
     when nil, 'local', :local, :serial, 'serial'
       Scout::Config.with_config do
@@ -207,7 +227,7 @@ class Workflow::LocalExecutor
         log = job_rules[:log] if job_rules
         log = Log.severity if log.nil?
         Log.with_severity log do
-          job.fork
+          job.fork(true)
         end
       end
     when 'batch', 'sched', 'slurm', 'pbs', 'lsf'

data/lib/scout/workflow/deployment/orchestrator/batches.rb

@@ -69,6 +69,7 @@ class Workflow::Orchestrator
         task_name = job.task_name
         task_rules = task_specific_rules(rules, workflow, task_name)
         acc = accumulate_rules(acc, task_rules.dup)
+        acc
       end
 
       if chain = batch[:chain]
@@ -92,12 +93,20 @@ class Workflow::Orchestrator
         next if batch[:deps].nil?
 
         if batch[:deps].any?
-          batch_dep_jobs = batch[:top_level].rec_dependencies
+          batch_dep_jobs = batch[:top_level].rec_dependencies.to_a
           target = batch[:deps].select do |target|
-            batch_dep_jobs.include?(target[:top_level]) &&
-              (batch[:deps] - [target] - target[:deps]).empty?
+            target_deps = []
+            stack = [target]
+            while stack.any?
+              c = stack.pop
+              target_deps << c
+              stack.concat c[:deps]
+            end
+            (batch[:deps] - target_deps).empty?
           end.first
           next if target.nil?
+          all_target_jobs = ([target] + target[:deps]).collect{|d| d[:jobs] }.flatten
+          next if all_target_jobs.reject{|j| batch_dep_jobs.include? j }.any?
           target[:jobs] = batch[:jobs] + target[:jobs]
           target[:deps] = (target[:deps] + batch[:deps]).uniq - [target]
           target[:top_level] = batch[:top_level]
@@ -119,12 +128,64 @@ class Workflow::Orchestrator
     jobs = [jobs] unless Array === jobs
 
     workload = job_workload(jobs)
-    job_chains_map = jobs.inject([]){|acc,job| acc.concat(self.job_chains(rules, job)) }
+    job_chain_list = []
 
-    batches = chain_batches(rules, job_chains_map, workload)
+    jobs.each do |job|
+      job_chains = self.job_chains(rules, job)
+      job_chains.each do |chain,list|
+        list.each do |info|
+          job_chain_list << [chain,info]
+        end
+      end
+    end
+
+    batches = chain_batches(rules, job_chain_list, workload)
     batches = add_batch_deps(batches)
     batches = add_rules_and_consolidate(rules, batches)
 
     batches
   end
+
+  def self.sort_batches(batches)
+    pending = batches.dup
+    sorted = []
+    while pending.any?
+      leaf_nodes = batches.select{|batch| batch[:deps].nil? || (batch[:deps] - sorted).empty? }
+      sorted.concat(leaf_nodes - sorted)
+      pending -= leaf_nodes
+    end
+    sorted
+  end
+
+  def self.errors_in_batch(batch)
+    errors = batch[:jobs].select do |job|
+      job.error? && ! job.recoverable_error?
+    end
+
+    errors.empty? ? false : errors
+  end
+
+  def self.clean_batches(batches)
+    error = []
+    batches.collect do |batch|
+      if failed = Workflow::Orchestrator.errors_in_batch(batch)
+        Log.warn "Batch contains errors #{batch[:top_level].short_path} #{Log.fingerprint failed}"
+        error << batch
+        next
+      elsif (error_deps = error & batch[:deps]).any?
+        if error_deps.reject{|b| b[:top_level].canfail? }.any?
+          Log.warn "Batch depends on batches with errors #{batch[:top_level].short_path} #{Log.fingerprint(error_deps.collect{|d| d[:top_level] })}"
+          error << batch
+          next
+        else
+          batch[:deps] -= error_deps
+        end
+      end
+      batch
+    end.compact
+  end
+
+  def self.inspect_batch(batch)
+    batch.merge(deps: batch[:deps].collect{|b| b[:top_level] })
+  end
 end