scout-gear 10.11.6 → 10.11.8

data/lib/scout/workflow/deployment/orchestrator/chains.rb

@@ -58,47 +58,64 @@ class Workflow::Orchestrator
     chains
   end
 
+  def self.add_chain(job_chains, match, info)
+    if job_chains[match]
+      current = job_chains[match]
+      new_info = {}
+      new_info[:jobs] = (current[:jobs] + info[:jobs]).uniq
+      if current[:top_level].rec_dependencies.include?(info[:top_level]) ||
+          current[:top_level].input_dependencies.include?(info[:top_level])
+        new_info[:top_level] = current[:top_level]
+      else
+        new_info[:top_level] = info[:top_level]
+      end
+      job_chains[match] = new_info
+    else
+      job_chains[match] = info
+    end
+  end
+
   def self.job_chains(rules, job, computed = {})
-    key = Log.fingerprint([rules, job.path, job.object_id])
+    chains = parse_chains(rules)
+    key = Log.fingerprint([job.path, job.object_id, chains])
     return computed[key] if computed.has_key?(key)
 
-    chains = parse_chains(rules)
-    matches = check_chains(chains, job)
-    dependencies = job_dependencies(job)
+    job_chains = check_chains(chains, job)
+    job_batches = {}
+    new_batches = {}
+    job_dependencies(job).each do |dep|
+      dep_chains = check_chains(chains, dep)
+      common_chains = job_chains & dep_chains
 
-    job_chains = []
-    new_job_chains = {}
-    dependencies.each do |dep|
-      dep_matches = check_chains(chains, dep)
-      common = matches & dep_matches
+      dep_batches = job_chains(rules, dep, computed)
 
-      dep_chains = job_chains(rules, dep, computed)
       found = []
-      dep_chains.each do |match, info|
-        if common.include?(match)
-          found << match
-          new_info = new_job_chains[match] ||= {}
-          new_info[:jobs] ||= []
-          new_info[:jobs].concat info[:jobs]
-          new_info[:top_level] = job
+      common_chains.each do |chain|
+        info = new_batches[chain]
+        info = {top_level: job, jobs: [job]} if info.nil?
+        if dep_batches[chain]
+          found << chain
+          dep_batches[chain].each do |dep_info|
+            info[:jobs] += dep_info[:jobs] - info[:jobs]
+          end
         else
-          job_chains << [match, info]
+          info[:jobs] << dep
         end
+        new_batches[chain] = info
       end
 
-      (common - found).each do |match|
-          info = {}
-          info[:jobs] = [job, dep]
-          info[:top_level] = job
-          job_chains << [match, info]
-        end
-      end
-
-      new_job_chains.each do |match, info|
-        info[:jobs].prepend job
-        job_chains << [match, info]
+      dep_batches.each do |chain,list|
+        next if found.include? chain
+        job_batches[chain] ||= []
+        job_batches[chain].concat list
       end
+    end
 
-      computed[key] = job_chains
+    new_batches.each do |match, info|
+      job_batches[match] ||= []
+      job_batches[match] << info
     end
+
+    computed[key] = job_batches
+  end
 end

data/lib/scout/workflow/deployment/orchestrator/rules.rb

@@ -47,7 +47,7 @@ class Workflow::Orchestrator
       case k.to_s
       when "config_keys"
         current[k] = add_config_keys current["config_keys"], value
-      when "cpus"
+      when "task_cpus", 'cpus'
         # choose max
         vals = [current[k], value].compact.map{|v| v.to_i }
         current[k] = vals.max unless vals.empty?
@@ -183,7 +183,7 @@ class Workflow::Orchestrator
     r = rules_block[:resources] || {}
     r = IndiferentHash.setup r
 
-    r = IndiferentHash.add_defaults r, 
+    r = IndiferentHash.add_defaults r,
       cpus: rules_block[:cpus] || rules_block[:task_cpus] || 1,
       time: rules_block[:time]
 
@@ -237,7 +237,7 @@ class Workflow::Orchestrator
       merge_rule_file(acc, file_rules)
     end
   end
-  
+
   def self.load_rules_for_job(jobs)
     jobs = [jobs] unless Array === jobs
 

data/lib/scout/workflow/deployment/orchestrator/workload.rb

@@ -1,32 +1,21 @@
 class Workflow::Orchestrator
 
-  def self.job_workload(jobs)
-    workload = []
-    path_jobs = {}
+  def self.prepare_for_execution(job)
+    rec_dependencies = job.rec_dependencies(true)
 
-    jobs = [jobs] unless Array === jobs
+    return if rec_dependencies.empty?
 
-    jobs.each do |job|
-      path_jobs[job.path] = job
-    end
+    all_deps = rec_dependencies + [job]
 
-    heap = []
-    heap += jobs.collect(&:path)
-    while job_path = heap.pop
-      j = path_jobs[job_path]
-      next if j.done?
-      workload << j
-
-      deps = job_dependencies(j)
-      deps.each do |d|
-        path_jobs[d.path] ||= d
+    all_deps.each do |dep|
+      begin
+        dep.clean if (dep.error? && dep.recoverable_error?) ||
+          dep.aborted? || (dep.done? && ! dep.updated?)
+      rescue RbbtException
+        Log.exception $!
+        next
       end
-
-      heap.concat deps.collect(&:path)
-      heap.uniq!
     end
-
-    path_jobs
   end
 
   def self.job_workload(jobs)

data/lib/scout/workflow/deployment/scheduler/job.rb

@@ -1,5 +1,5 @@
 module SchedulerJob
-  @batch_base_dir = File.expand_path(File.join('~/scout-batch')) 
+  @batch_base_dir = File.expand_path(File.join('~/scout-batch'))
   self.singleton_class.attr_accessor :batch_base_dir
 
   module_function
@@ -27,7 +27,7 @@ module SchedulerJob
 
       singularity_img, singularity_opt_dir, singularity_ruby_inline, singularity_mounts = options.values_at :singularity_img, :singularity_opt_dir, :singularity_ruby_inline, :singularity_mounts
 
-      singularity_cmd = %(singularity exec -e -B "#{File.expand_path singularity_opt_dir}":/singularity_opt/ -B "#{File.expand_path singularity_ruby_inline}":"/.singularity_ruby_inline":rw ) 
+      singularity_cmd = %(singularity exec -e -B "#{File.expand_path singularity_opt_dir}":/singularity_opt/ -B "#{File.expand_path singularity_ruby_inline}":"/.singularity_ruby_inline":rw )
 
       if singularity_mounts
         singularity_mounts.split(",").each do |mount|
@@ -37,7 +37,7 @@ module SchedulerJob
 
       if contain && options[:hardened]
         singularity_cmd << %( -C -H "#{contain}" \
--B "/.singularity_ruby_inline":"#{contain}/.singularity_ruby_inline":rw 
+-B "/.singularity_ruby_inline":"#{contain}/.singularity_ruby_inline":rw
 -B "#{options[:batch_dir]}" \
 -B /scratch/tmp \
           #{ group != user_group ? "-B /gpfs/projects/#{user_group}" : "" } \
@@ -81,17 +81,15 @@ module SchedulerJob
 
     task = job.task_name
 
-    if job.recursive_overriden_deps.any?
-      override_deps = job.recursive_overriden_deps.
-        select do |dep| Symbol === dep.overriden end.
-        collect do |dep| 
-          o_workflow = dep.overriden_workflow || dep.workflow
-          o_workflow = o_workflow.name if o_workflow.respond_to?(:name)
-          o_task_name = dep.overriden_task || dep.task.name
-          name = [o_workflow, o_task_name] * "#"
-          [name, dep.path] * "="  
-        end.uniq * ","
-        options[:override_deps] = override_deps unless override_deps.empty?
+    if job.overriden?
+      override_deps = job.recursive_overrider_deps.collect do |dep|
+        o_workflow = dep.overriden_workflow || dep.workflow
+        o_workflow = o_workflow.name if o_workflow.respond_to?(:name)
+        o_task_name = dep.overriden_task || dep.task.name
+        name = [o_workflow, o_task_name] * "#"
+        [name, dep.path] * "="
+      end.uniq * ","
+      options[:override_deps] = override_deps unless override_deps.empty?
     end
 
     # Save inputs into inputs_dir (only if provided)
@@ -192,7 +190,7 @@ workflow task #{workflow} #{task} #{cmds}
     keys_from_config.each do |key|
       next unless batch_options.include? key
       default_value = Scout::Config.get(key, "batch_#{key}", "batch")
-      next if default_value.nil? 
+      next if default_value.nil?
       IndiferentHash.add_defaults batch_options, default_value
     end
 
@@ -211,7 +209,7 @@ workflow task #{workflow} #{task} #{cmds}
         batch_options[:contain] = File.join(contain_base, random_file)
       end
 
-      batch_options[:sync] ||= "~/.scout/var/jobs" 
+      batch_options[:sync] ||= "~/.scout/var/jobs"
       batch_options[:wipe_container] ||= 'post'
     end
 
@@ -219,23 +217,23 @@ workflow task #{workflow} #{task} #{cmds}
       options[:workdir_all] = batch_options[:contain]
     end
 
-    IndiferentHash.add_defaults batch_options, 
+    IndiferentHash.add_defaults batch_options,
       :batch_name => batch_name,
-      :inputs_dir => inputs_dir, 
-      :nodes => 1, 
+      :inputs_dir => inputs_dir,
+      :nodes => 1,
       :step_path => job.path,
       :task_cpus => 1,
-      :time => '2min', 
+      :time => '2min',
       :env => {'JDK_JAVA_OPTIONS' => "-Xms1g -Xmx${MAX_MEMORY}m"},
       :singularity_img => ENV["SINGULARITY_IMG"] || "~/scout.singularity.img",
       :singularity_ruby_inline => ENV["SINGULARITY_RUBY_INLINE"] || "~/.singularity_ruby_inline",
       :singularity_opt_dir => ENV["SINGULARITY_OPT_DIR"] || "~/singularity_opt",
-      :workdir => Dir.pwd 
+      :workdir => Dir.pwd
 
     exec_cmd = exec_cmd(job, batch_options)
     scout_cmd = scout_job_exec_cmd(job, options)
 
-    IndiferentHash.add_defaults batch_options, 
+    IndiferentHash.add_defaults batch_options,
       :exec_cmd => exec_cmd,
       :scout_cmd => scout_cmd
 
@@ -344,7 +342,7 @@ batch_erase_contain_dir()
 function batch_sync_contain_dir(){
   mkdir -p "$(dirname '#{sync}')"
   rsync -avztAXHP --copy-unsafe-links "#{source}/" "#{sync}/" 2>1 >> '#{options[:fsync]}'
-  sync_es="$?" 
+  sync_es="$?"
   echo $sync_es > '#{options[:fsexit]}'
   find '#{sync}' -type l -ls | awk '$13 ~ /^#{sync.gsub('/','\/')}/ { sub("#{source}", "#{sync}", $13); print $11, $13 }' | while read A B; do rm $A; ln -s $B $A; done
 }
@@ -353,7 +351,7 @@ function batch_sync_contain_dir(){
 
     if options[:env]
       prepare_environment +=<<-EOF
-# Set ENV variables 
+# Set ENV variables
 #{options[:env].collect{|n,v| "export #{n}=\"#{v}\"" } * "\n"}
       EOF
     end
@@ -384,7 +382,7 @@ for tmpd in persist_locks  produce_locks  R_sockets  sensiblewrite  sensiblewrit
     mkdir -p "#{contain}/.scout/tmp/$tmpd"
 done
 
-# Copy environment 
+# Copy environment
 cp ~/.scout/etc/environment #{contain}/.scout/etc/
 
 # Set search_paths
@@ -406,7 +404,7 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
     exec_cmd, job_cmd, task_cpus = options.values_at :exec_cmd, :scout_cmd, :task_cpus
 
     script=<<-EOF
-step_path=$( 
+step_path=$(
       #{exec_cmd} #{job_cmd} --printpath
 )
 exit_status=$?
@@ -426,7 +424,7 @@ fi
 
     if options[:sync]
       sync_environment +=<<-EOF
-if [ $exit_status == '0' ]; then 
+if [ $exit_status == '0' ]; then
   batch_sync_contain_dir
 else
   sync_es=$exit_status
@@ -441,8 +439,8 @@ fi
     cleanup_environment = ""
 
     cleanup_environment +=<<-EOF if options[:purge_deps]
-if [ $exit_status == '0' ]; then 
-  #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}' 
+if [ $exit_status == '0' ]; then
+  #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}'
 fi
     EOF
 
@@ -453,7 +451,7 @@ batch_erase_contain_dir
         EOF
       elsif options[:wipe_container] == 'post' || options[:wipe_container] == 'both'
         cleanup_environment +=<<-EOF
-if [ $sync_es == '0' -a $empty_contain_dir == 'true' ]; then 
+if [ $sync_es == '0' -a $empty_contain_dir == 'true' ]; then
   batch_erase_contain_dir
 fi
         EOF
@@ -512,7 +510,7 @@ exit $exit_status
 env > #{batch_options[:fenv]}
 
 # #{Log.color :green, "2. Execute"}
-#{execute} 
+#{execute}
 
 # #{Log.color :green, "3. Sync and cleanup environment"}
 #{sync_environment}
@@ -555,13 +553,13 @@ env > #{batch_options[:fenv]}
   def run_job(job, options = {})
     system = self.to_s.split("::").last
 
-    batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run, orchestration_rules_file = IndiferentHash.process_options options, 
+    batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run, orchestration_rules_file = IndiferentHash.process_options options,
       :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run, :orchestration_rules,
       :batch_base_dir => SchedulerJob.batch_base_dir
 
     if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
       Log.info "Job #{job.short_path} already queued in #{batch_job}"
-      return batch_job, batch_dir_for_id(batch_base_dir, batch_job) 
+      return batch_job, batch_dir_for_id(batch_base_dir, batch_job)
     end
 
     if job.running?
@@ -570,7 +568,7 @@ env > #{batch_options[:fenv]}
       if job.info[:batch_job]
         return job.info[:batch_job], batch_dir_for_id(batch_base_dir, batch_job)
       else
-        return 
+        return
       end
     end
 
@@ -582,8 +580,8 @@ env > #{batch_options[:fenv]}
     workflows_to_load = job.rec_dependencies.select{|d| Step === d}.collect{|d| d.workflow }.compact.collect(&:to_s) - [workflow.to_s]
 
     TmpFile.with_file(nil, remove_batch_dir, :tmpdir => batch_base_dir, :prefix => "#{system}_scout_job-#{workflow.to_s}-#{task_name}-") do |batch_dir|
-      IndiferentHash.add_defaults options, 
-        :batch_dir => batch_dir, 
+      IndiferentHash.add_defaults options,
+        :batch_dir => batch_dir,
         :inputs_dir => File.join(batch_dir, "inputs_dir"),
         :workflows => workflows_to_load.any? ? workflows_to_load.uniq * "," : nil
 

data/lib/scout/workflow/deployment/scheduler/lfs.rb

@@ -1,7 +1,7 @@
 require_relative 'job'
 require 'scout'
 
-module LSF 
+module LSF
   extend SchedulerJob
 
   def self.system

data/lib/scout/workflow/deployment/scheduler/pbs.rb

@@ -1,7 +1,7 @@
 require_relative 'job'
 require 'scout'
 
-module PBS 
+module PBS
   extend SchedulerJob
 
   def self.system
@@ -13,7 +13,7 @@ module PBS
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
 let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $PBS_CPUS_PER_TASK )"
 MAX_MEMORY="$MAX_MEMORY_DEFAULT"
-[ ! -z $PBS_MEM_PER_CPU ] && let MAX_MEMORY="$PBS_MEM_PER_CPU * $PBS_CPUS_PER_TASK" 
+[ ! -z $PBS_MEM_PER_CPU ] && let MAX_MEMORY="$PBS_MEM_PER_CPU * $PBS_CPUS_PER_TASK"
 [ ! -z $PBS_MEM_PER_NODE ] && MAX_MEMORY="$PBS_MEM_PER_NODE"
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY
@@ -36,7 +36,7 @@ cd ${PBS_O_WORKDIR}
     time       = IndiferentHash.process_options options, :time
     nodes      = IndiferentHash.process_options options, :nodes
 
-    # PBS 
+    # PBS
     place      = IndiferentHash.process_options options, :place, :place => 'scatter'
     system     = IndiferentHash.process_options options, :partition
     filesystems = IndiferentHash.process_options options, :filesystems
@@ -45,7 +45,7 @@ cd ${PBS_O_WORKDIR}
 
     filesystems = filesystems * "," if Array === filesystems
 
-    # NOT USED 
+    # NOT USED
     partition  = IndiferentHash.process_options options, :partition
     task_cpus  = IndiferentHash.process_options options, :task_cpus
     exclusive  = IndiferentHash.process_options options, :exclusive

data/lib/scout/workflow/deployment/scheduler/slurm.rb

@@ -1,7 +1,7 @@
 require_relative 'job'
 require 'scout'
 
-module SLURM 
+module SLURM
 
   extend SchedulerJob
 
@@ -15,7 +15,7 @@ module SLURM
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
 let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $SLURM_CPUS_PER_TASK )"
 MAX_MEMORY="$MAX_MEMORY_DEFAULT"
-[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" 
+[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK"
 [ ! -z $SLURM_MEM_PER_NODE ] && MAX_MEMORY="$SLURM_MEM_PER_NODE"
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY

data/lib/scout/workflow/deployment/scheduler.rb

@@ -12,31 +12,42 @@ module Workflow::Scheduler
     Workflow::Scheduler.process_batches(batches, options)
   end
 
+  def self.produce_single_batch(job, rules = {}, options = {})
+    batches = Workflow::Orchestrator.job_batches(rules, job)
+    rules = batches.inject({}){|acc,b| acc = Workflow::Orchestrator.accumulate_rules acc, b[:rules] }
+
+    max_time = batches.collect{|b| b[:rules][:time] }.max
+    rules[:time] = options.include?(:time) ? options[:time] : max_time
+    rules[:deploy] = :local
+    if rules[:exclusive]
+      rules[:task_cpus] = nil
+      rules[:cpus] = nil
+    end
+
+    batch = {top_level: job, jobs: [job], rules: rules, deps: [] }
+    Workflow::Scheduler.process_batches([batch], options)
+  end
+
+
   def self.process_batches(batches, process_options = {})
     failed_jobs = []
 
-    pending = batches.dup
-
-    sorted = []
-    while pending.any?
-      leaf_nodes = batches.select{|batch| (batch[:deps] - sorted).empty? }
-      sorted.concat(leaf_nodes - sorted)
-      pending -= leaf_nodes
-    end
+    sorted = Workflow::Orchestrator.sort_batches batches
+    cleaned = Workflow::Orchestrator.clean_batches sorted
 
     batch_system = Scout::Config.get :system, :batch, :scheduler, 'env:BATCH_SYSTEM', default: 'SLURM'
 
     batch_ids = {}
-    sorted.collect do |batch|
+    cleaned.collect do |batch|
       job_options = batch[:rules]
       job_options = IndiferentHash.add_defaults job_options, process_options.dup
 
       if batch[:deps].nil?
-        batch_dependencies = [] 
-      else 
+        batch_dependencies = []
+      else
         top_jobs = batch[:jobs]
 
-        batch_dependencies = batch[:deps].collect{|dep| 
+        batch_dependencies = batch[:deps].collect{|dep|
           dep_target = dep[:top_level]
           id = batch_ids[dep_target].to_s
 

data/lib/scout/workflow/deployment/trace.rb

@@ -132,10 +132,10 @@ module Workflow
       info[:time] << time
 
       report_keys.each do |key|
-        info[key] = dep_info[key] 
+        info[key] = dep_info[key]
       end
 
-      dep.info[:config_keys].each do |kinfo| 
+      dep.info[:config_keys].each do |kinfo|
         key, value, tokens = kinfo
 
         info[key.to_s] = value if report_keys.include? key.to_s

data/lib/scout/workflow/documentation.rb

@@ -31,7 +31,7 @@ module Workflow
     title = doc_parse_first_line doc
     description, task_info = doc_parse_up_to doc, /^# Tasks/i
     task_description, tasks = doc_parse_up_to task_info, /^##/, true
-    tasks = doc_parse_chunks tasks, /^## (.*)/ 
+    tasks = doc_parse_chunks tasks, /^## (.*)/
     {:title => title.strip, :description => description.strip, :task_description => task_description.strip, :tasks => tasks}
   end
 
@@ -52,11 +52,11 @@ module Workflow
                          documentation = Workflow.parse_workflow_doc documentation_markdown
 
                          if @description && (documentation[:description].nil? || documentation[:description].empty?)
-                           documentation[:description] = @description 
+                           documentation[:description] = @description
                          end
 
                          if @title && (documentation[:title].nil? || documentation[:title].empty?)
-                           documentation[:title] = @title 
+                           documentation[:title] = @title
                          end
                          documentation[:tasks].each do |task, description|
                            if task.include? "#"
@@ -74,7 +74,7 @@ module Workflow
                            if workflow.tasks.include? task
                              workflow.tasks[task].description = description
                            else
-                             Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
+                             Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task"
                            end
                          end
                          documentation

data/lib/scout/workflow/export.rb

@@ -39,7 +39,7 @@ module Workflow
     asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
     stream_exports.replace stream_exports - names if stream_exports
   end
-  
+
   def export_exec(*names)
     unexport *names
     exec_exports.concat names

data/lib/scout/workflow/path.rb

@@ -11,9 +11,9 @@ module Path
         _loaded = false
         begin
           Kernel.const_get(workflow)
-        rescue 
+        rescue
           if ! _loaded
-            Workflow.require_workflow workflow 
+            Workflow.require_workflow workflow
             _loaded = true
             retry
           end

data/lib/scout/workflow/step/children.rb

@@ -40,7 +40,7 @@ class Step
       if c == "\n"
         Log.logn "STDOUT [#{child_pid}]: ", level
       end
-    end 
+    end
 
     io.join
 

data/lib/scout/workflow/step/dependencies.rb

@@ -1,4 +1,5 @@
 class Step
+
   def rec_dependencies(connected = false, seen = Set.new)
     @rec_dependencies = {}
     @rec_dependencies[connected] ||= begin
@@ -34,6 +35,31 @@ class Step
     end.compact.uniq
   end
 
+  def overrider?
+    ! (overriden_task.nil? && overriden_workflow.nil?)
+  end
+
+  def overriden?
+    @overriden
+  end
+
+  def overriden_deps
+    dependencies.select{|d| d.overriden? }
+  end
+
+  def recursive_overriden_deps
+    overriden_deps = self.overriden_deps
+    overriden_deps + overriden_deps.collect{|dep| dep.recursive_overriden_deps }
+  end
+
+  def overrider_deps
+    dependencies.select{|d| d.overrider? }
+  end
+
+  def recursive_overrider_deps
+    self.rec_dependencies.select{|dep| dep.overrider? }
+  end
+
   def prepare_dependencies
     inverse_dep = {}
 
@@ -44,7 +70,14 @@ class Step
       end
 
       next if dep.done?
-      next if dep.error? && ! dep.recoverable_error?
+
+      if dep.error? && ! dep.recoverable_error?
+        if dep.canfail?
+          next
+        else
+          raise dep.exception
+        end
+      end
 
       if dep.dependencies
         dep.dependencies.each do |d|
@@ -74,7 +107,7 @@ class Step
   end
 
   def run_dependencies
-    all_dependencies.each do |dep| 
+    all_dependencies.each do |dep|
       next if dep.running? || dep.done?
       next if dep.error? && ! dep.recoverable_error?
 
@@ -100,7 +133,7 @@ class Step
   end
 
   def abort_dependencies
-    all_dependencies.each{|dep| dep.abort if dep.running? } 
+    all_dependencies.each{|dep| dep.abort if dep.running? }
   end
 
   def self.wait_for_jobs(jobs, canfail=false)