scout-gear 10.11.4 → 10.11.7

data/lib/scout/workflow/deployment/local.rb

@@ -42,86 +42,95 @@ class Workflow::LocalExecutor
     @available_resources = IndiferentHash.setup(available_resources)
     @resources_requested = IndiferentHash.setup({})
     @resources_used      = IndiferentHash.setup({})
+    Log.info "LocalExecutor initiated #{Log.fingerprint available_resources}"
   end
 
-  def process_batches(batches)
+  def process_batches(batches, bar: true)
     retry_jobs = []
     failed_jobs = []
 
-    while batches.reject{|b| Workflow::Orchestrator.done_batch?(b) }.any?
+    bar = {desc: "Processing batches"} if TrueClass === bar
+    bar = {bar: bar} if Log::ProgressBar === bar
+    Log::ProgressBar.with_bar batches.length, bar do |bar|
+      bar.init if bar
 
-      candidates = Workflow::LocalExecutor.candidates(batches)
-      top_level_jobs = candidates.collect{|batch| batch[:top_level] }
+      while (missing_batches = batches.reject{|b| Workflow::Orchestrator.done_batch?(b) }).any?
 
-      raise NoWork, "No candidates and no running jobs #{Log.fingerprint batches}" if resources_used.empty? && top_level_jobs.empty?
+        bar.pos batches.select{|b| Workflow::Orchestrator.done_batch?(b) }.length if bar
 
-      if candidates.reject{|batch| failed_jobs.include? batch[:top_level] }.empty? && resources_used.empty? && top_level_jobs.empty?
-        exception = failed_jobs.collect(&:get_exception).compact.first
-        if exception
-          Log.warn 'Some work failed'
-          raise exception 
-        else
-          raise 'Some work failed'
-        end
-      end
+        candidates = Workflow::LocalExecutor.candidates(batches)
+        top_level_jobs = candidates.collect{|batch| batch[:top_level] }
 
-      candidates.each do |batch|
-        begin
+        raise NoWork, "No candidates and no running jobs #{Log.fingerprint batches}" if resources_used.empty? && top_level_jobs.empty?
 
-          job = batch[:top_level]
+        if candidates.reject{|batch| failed_jobs.include? batch[:top_level] }.empty? && resources_used.empty? && top_level_jobs.empty?
+          exception = failed_jobs.collect(&:get_exception).compact.first
+          if exception
+            Log.warn 'Some work failed'
+            raise exception
+          else
+            raise 'Some work failed'
+          end
+        end
 
-          case
-          when (job.error? || job.aborted?)
-            begin
-              if job.recoverable_error?
-                if retry_jobs.include?(job)
-                  Log.warn "Failed twice #{job.path} with recoverable error"
-                  retry_jobs.delete job
+        candidates.each do |batch|
+          begin
+
+            job = batch[:top_level]
+
+            case
+            when (job.error? || job.aborted?)
+              begin
+                if job.recoverable_error?
+                  if retry_jobs.include?(job)
+                    Log.warn "Failed twice #{job.path} with recoverable error"
+                    retry_jobs.delete job
+                    failed_jobs << job
+                    next
+                  else
+                    retry_jobs << job
+                    job.clean
+                    raise TryAgain
+                  end
+                else
                   failed_jobs << job
+                  Log.warn "Non-recoverable error in #{job.path}"
                   next
-                else
-                  retry_jobs << job
-                  job.clean
-                  raise TryAgain
                 end
-              else
-                failed_jobs << job
-                Log.warn "Non-recoverable error in #{job.path}"
-                next
+              ensure
+                Log.warn "Releases resources from failed job: #{job.path}"
+                release_resources(job)
               end
-            ensure
-              Log.warn "Releases resources from failed job: #{job.path}"
+            when job.done?
+              Log.debug "Orchestrator done #{job.path}"
               release_resources(job)
+              clear_batch(batches, batch)
+              erase_job_dependencies(job, batches)
+            when job.running?
+              next
+
+            else
+              check_resources(batch) do
+                run_batch(batch)
+              end
             end
-          when job.done?
-            Log.debug "Orchestrator done #{job.path}"
-            release_resources(job)
+          rescue TryAgain
+            retry
+          end
+        end
+
+        batches.each do |batch|
+          job = batch[:top_level]
+          if job.done? || job.aborted? || job.error?
+            job.join if job.done?
             clear_batch(batches, batch)
+            release_resources(job)
             erase_job_dependencies(job, batches)
-          when job.running?
-            next
-
-          else
-            check_resources(batch) do
-              run_batch(batch)
-            end
           end
-        rescue TryAgain
-          retry
         end
-      end
 
-      batches.each do |batch|
-        job = batch[:top_level]
-        if job.done? || job.aborted? || job.error?
-          job.join if job.done?
-          clear_batch(batches, batch)
-          release_resources(job)
-          erase_job_dependencies(job, batches)
-        end
+        sleep timer
       end
-
-      sleep timer
     end
 
     batches.each{|batch|
@@ -141,7 +150,16 @@ class Workflow::LocalExecutor
 
   def process(rules, jobs = nil)
     jobs, rules = rules, {} if jobs.nil?
-    jobs = [jobs] if Step === jobs
+
+    if Step === jobs
+      jobs = [jobs]
+    end
+
+    if jobs.length == 1
+      bar = jobs.first.progress_bar("Process batches for #{jobs.first.short_path}")
+    else
+      bar = true
+    end
 
     batches = Workflow::Orchestrator.job_batches(rules, jobs)
     batches.each do |batch|
@@ -153,13 +171,13 @@ class Workflow::LocalExecutor
       batch[:rules] = rules
     end
 
-    process_batches(batches)
+    process_batches(batches, bar: bar)
   end
 
   def release_resources(job)
     if resources_used[job]
       Log.debug "Orchestrator releasing resouces from #{job.path}"
-      resources_used[job].each do |resource,value| 
+      resources_used[job].each do |resource,value|
         next if resource == 'size'
         resources_requested[resource] -= value.to_i
       end
@@ -195,8 +213,9 @@ class Workflow::LocalExecutor
   def run_batch(batch)
     job, job_rules = batch.values_at :top_level, :rules
 
-    rules = batch[:rules] 
+    rules = batch[:rules]
     deploy = rules[:deploy] if rules
+    Log.debug "Processing #{deploy} #{job.short_path} #{Log.fingerprint job_rules}"
     case deploy
     when nil, 'local', :local, :serial, 'serial'
       Scout::Config.with_config do
@@ -207,7 +226,7 @@ class Workflow::LocalExecutor
         log = job_rules[:log] if job_rules
         log = Log.severity if log.nil?
         Log.with_severity log do
-          job.fork
+          job.fork(true)
         end
       end
     when 'batch', 'sched', 'slurm', 'pbs', 'lsf'

data/lib/scout/workflow/deployment/orchestrator/batches.rb

@@ -69,6 +69,7 @@ class Workflow::Orchestrator
         task_name = job.task_name
         task_rules = task_specific_rules(rules, workflow, task_name)
         acc = accumulate_rules(acc, task_rules.dup)
+        acc
       end
 
       if chain = batch[:chain]
@@ -92,12 +93,20 @@ class Workflow::Orchestrator
         next if batch[:deps].nil?
 
         if batch[:deps].any?
-          batch_dep_jobs = batch[:top_level].rec_dependencies
+          batch_dep_jobs = batch[:top_level].rec_dependencies.to_a
           target = batch[:deps].select do |target|
-            batch_dep_jobs.include?(target[:top_level]) &&
-              (batch[:deps] - [target] - target[:deps]).empty?
+            target_deps = []
+            stack = [target]
+            while stack.any?
+              c = stack.pop
+              target_deps << c
+              stack.concat c[:deps]
+            end
+            (batch[:deps] - target_deps).empty?
           end.first
           next if target.nil?
+          all_target_jobs = ([target] + target[:deps]).collect{|d| d[:jobs] }.flatten
+          next if all_target_jobs.reject{|j| batch_dep_jobs.include? j }.any?
           target[:jobs] = batch[:jobs] + target[:jobs]
           target[:deps] = (target[:deps] + batch[:deps]).uniq - [target]
           target[:top_level] = batch[:top_level]
@@ -119,12 +128,64 @@ class Workflow::Orchestrator
     jobs = [jobs] unless Array === jobs
 
     workload = job_workload(jobs)
-    job_chains_map = jobs.inject([]){|acc,job| acc.concat(self.job_chains(rules, job)) }
+    job_chain_list = []
 
-    batches = chain_batches(rules, job_chains_map, workload)
+    jobs.each do |job|
+      job_chains = self.job_chains(rules, job)
+      job_chains.each do |chain,list|
+        list.each do |info|
+          job_chain_list << [chain,info]
+        end
+      end
+    end
+
+    batches = chain_batches(rules, job_chain_list, workload)
     batches = add_batch_deps(batches)
     batches = add_rules_and_consolidate(rules, batches)
 
     batches
   end
+
+  def self.sort_batches(batches)
+    pending = batches.dup
+    sorted = []
+    while pending.any?
+      leaf_nodes = batches.select{|batch| batch[:deps].nil? || (batch[:deps] - sorted).empty? }
+      sorted.concat(leaf_nodes - sorted)
+      pending -= leaf_nodes
+    end
+    sorted
+  end
+
+  def self.errors_in_batch(batch)
+    errors = batch[:jobs].select do |job|
+      job.error? && ! job.recoverable_error?
+    end
+
+    errors.empty? ? false : errors
+  end
+
+  def self.clean_batches(batches)
+    error = []
+    batches.collect do |batch|
+      if failed = Workflow::Orchestrator.errors_in_batch(batch)
+        Log.warn "Batch contains errors #{batch[:top_level].short_path} #{Log.fingerprint failed}"
+        error << batch
+        next
+      elsif (error_deps = error & batch[:deps]).any?
+        if error_deps.reject{|b| b[:top_level].canfail? }.any?
+          Log.warn "Batch depends on batches with errors #{batch[:top_level].short_path} #{Log.fingerprint(error_deps.collect{|d| d[:top_level] })}"
+          error << batch
+          next
+        else
+          batch[:deps] -= error_deps
+        end
+      end
+      batch
+    end.compact
+  end
+
+  def self.inspect_batch(batch)
+    batch.merge(deps: batch[:deps].collect{|b| b[:top_level] })
+  end
 end

data/lib/scout/workflow/deployment/orchestrator/chains.rb

@@ -58,47 +58,64 @@ class Workflow::Orchestrator
     chains
   end
 
+  def self.add_chain(job_chains, match, info)
+    if job_chains[match]
+      current = job_chains[match]
+      new_info = {}
+      new_info[:jobs] = (current[:jobs] + info[:jobs]).uniq
+      if current[:top_level].rec_dependencies.include?(info[:top_level]) ||
+          current[:top_level].input_dependencies.include?(info[:top_level])
+        new_info[:top_level] = current[:top_level]
+      else
+        new_info[:top_level] = info[:top_level]
+      end
+      job_chains[match] = new_info
+    else
+      job_chains[match] = info
+    end
+  end
+
   def self.job_chains(rules, job, computed = {})
-    key = Log.fingerprint([rules, job.path, job.object_id])
+    chains = parse_chains(rules)
+    key = Log.fingerprint([job.path, job.object_id, chains])
     return computed[key] if computed.has_key?(key)
 
-    chains = parse_chains(rules)
-    matches = check_chains(chains, job)
-    dependencies = job_dependencies(job)
+    job_chains = check_chains(chains, job)
+    job_batches = {}
+    new_batches = {}
+    job_dependencies(job).each do |dep|
+      dep_chains = check_chains(chains, dep)
+      common_chains = job_chains & dep_chains
 
-    job_chains = []
-    new_job_chains = {}
-    dependencies.each do |dep|
-      dep_matches = check_chains(chains, dep)
-      common = matches & dep_matches
+      dep_batches = job_chains(rules, dep, computed)
 
-      dep_chains = job_chains(rules, dep, computed)
       found = []
-      dep_chains.each do |match, info|
-        if common.include?(match)
-          found << match
-          new_info = new_job_chains[match] ||= {}
-          new_info[:jobs] ||= []
-          new_info[:jobs].concat info[:jobs]
-          new_info[:top_level] = job
+      common_chains.each do |chain|
+        info = new_batches[chain]
+        info = {top_level: job, jobs: [job]} if info.nil?
+        if dep_batches[chain]
+          found << chain
+          dep_batches[chain].each do |dep_info|
+            info[:jobs] += dep_info[:jobs] - info[:jobs]
+          end
         else
-          job_chains << [match, info]
+          info[:jobs] << dep
         end
+        new_batches[chain] = info
       end
 
-      (common - found).each do |match|
-          info = {}
-          info[:jobs] = [job, dep]
-          info[:top_level] = job
-          job_chains << [match, info]
-        end
-      end
-
-      new_job_chains.each do |match, info|
-        info[:jobs].prepend job
-        job_chains << [match, info]
+      dep_batches.each do |chain,list|
+        next if found.include? chain
+        job_batches[chain] ||= []
+        job_batches[chain].concat list
       end
+    end
 
-      computed[key] = job_chains
+    new_batches.each do |match, info|
+      job_batches[match] ||= []
+      job_batches[match] << info
     end
+
+    computed[key] = job_batches
+  end
 end

data/lib/scout/workflow/deployment/orchestrator/rules.rb

@@ -47,7 +47,7 @@ class Workflow::Orchestrator
       case k.to_s
       when "config_keys"
         current[k] = add_config_keys current["config_keys"], value
-      when "cpus"
+      when "task_cpus", 'cpus'
         # choose max
         vals = [current[k], value].compact.map{|v| v.to_i }
         current[k] = vals.max unless vals.empty?
@@ -183,7 +183,7 @@ class Workflow::Orchestrator
     r = rules_block[:resources] || {}
     r = IndiferentHash.setup r
 
-    r = IndiferentHash.add_defaults r, 
+    r = IndiferentHash.add_defaults r,
       cpus: rules_block[:cpus] || rules_block[:task_cpus] || 1,
       time: rules_block[:time]
 
@@ -237,7 +237,7 @@ class Workflow::Orchestrator
       merge_rule_file(acc, file_rules)
     end
   end
-  
+
   def self.load_rules_for_job(jobs)
     jobs = [jobs] unless Array === jobs
 

data/lib/scout/workflow/deployment/orchestrator/workload.rb

@@ -1,32 +1,21 @@
 class Workflow::Orchestrator
 
-  def self.job_workload(jobs)
-    workload = []
-    path_jobs = {}
+  def self.prepare_for_execution(job)
+    rec_dependencies = job.rec_dependencies(true)
 
-    jobs = [jobs] unless Array === jobs
+    return if rec_dependencies.empty?
 
-    jobs.each do |job|
-      path_jobs[job.path] = job
-    end
+    all_deps = rec_dependencies + [job]
 
-    heap = []
-    heap += jobs.collect(&:path)
-    while job_path = heap.pop
-      j = path_jobs[job_path]
-      next if j.done?
-      workload << j
-
-      deps = job_dependencies(j)
-      deps.each do |d|
-        path_jobs[d.path] ||= d
+    all_deps.each do |dep|
+      begin
+        dep.clean if (dep.error? && dep.recoverable_error?) ||
+          dep.aborted? || (dep.done? && ! dep.updated?)
+      rescue RbbtException
+        Log.exception $!
+        next
       end
-
-      heap.concat deps.collect(&:path)
-      heap.uniq!
     end
-
-    path_jobs
   end
 
   def self.job_workload(jobs)

data/lib/scout/workflow/deployment/scheduler/job.rb

@@ -1,5 +1,5 @@
 module SchedulerJob
-  @batch_base_dir = File.expand_path(File.join('~/scout-batch')) 
+  @batch_base_dir = File.expand_path(File.join('~/scout-batch'))
   self.singleton_class.attr_accessor :batch_base_dir
 
   module_function
@@ -27,7 +27,7 @@ module SchedulerJob
 
       singularity_img, singularity_opt_dir, singularity_ruby_inline, singularity_mounts = options.values_at :singularity_img, :singularity_opt_dir, :singularity_ruby_inline, :singularity_mounts
 
-      singularity_cmd = %(singularity exec -e -B "#{File.expand_path singularity_opt_dir}":/singularity_opt/ -B "#{File.expand_path singularity_ruby_inline}":"/.singularity_ruby_inline":rw ) 
+      singularity_cmd = %(singularity exec -e -B "#{File.expand_path singularity_opt_dir}":/singularity_opt/ -B "#{File.expand_path singularity_ruby_inline}":"/.singularity_ruby_inline":rw )
 
       if singularity_mounts
         singularity_mounts.split(",").each do |mount|
@@ -37,7 +37,7 @@ module SchedulerJob
 
       if contain && options[:hardened]
         singularity_cmd << %( -C -H "#{contain}" \
--B "/.singularity_ruby_inline":"#{contain}/.singularity_ruby_inline":rw 
+-B "/.singularity_ruby_inline":"#{contain}/.singularity_ruby_inline":rw
 -B "#{options[:batch_dir]}" \
 -B /scratch/tmp \
           #{ group != user_group ? "-B /gpfs/projects/#{user_group}" : "" } \
@@ -81,17 +81,15 @@ module SchedulerJob
 
     task = job.task_name
 
-    if job.recursive_overriden_deps.any?
-      override_deps = job.recursive_overriden_deps.
-        select do |dep| Symbol === dep.overriden end.
-        collect do |dep| 
-          o_workflow = dep.overriden_workflow || dep.workflow
-          o_workflow = o_workflow.name if o_workflow.respond_to?(:name)
-          o_task_name = dep.overriden_task || dep.task.name
-          name = [o_workflow, o_task_name] * "#"
-          [name, dep.path] * "="  
-        end.uniq * ","
-        options[:override_deps] = override_deps unless override_deps.empty?
+    if job.overriden?
+      override_deps = job.recursive_overrider_deps.collect do |dep|
+        o_workflow = dep.overriden_workflow || dep.workflow
+        o_workflow = o_workflow.name if o_workflow.respond_to?(:name)
+        o_task_name = dep.overriden_task || dep.task.name
+        name = [o_workflow, o_task_name] * "#"
+        [name, dep.path] * "="
+      end.uniq * ","
+      options[:override_deps] = override_deps unless override_deps.empty?
     end
 
     # Save inputs into inputs_dir (only if provided)
@@ -192,7 +190,7 @@ workflow task #{workflow} #{task} #{cmds}
     keys_from_config.each do |key|
       next unless batch_options.include? key
       default_value = Scout::Config.get(key, "batch_#{key}", "batch")
-      next if default_value.nil? 
+      next if default_value.nil?
       IndiferentHash.add_defaults batch_options, default_value
     end
 
@@ -211,7 +209,7 @@ workflow task #{workflow} #{task} #{cmds}
         batch_options[:contain] = File.join(contain_base, random_file)
       end
 
-      batch_options[:sync] ||= "~/.scout/var/jobs" 
+      batch_options[:sync] ||= "~/.scout/var/jobs"
       batch_options[:wipe_container] ||= 'post'
     end
 
@@ -219,23 +217,23 @@ workflow task #{workflow} #{task} #{cmds}
       options[:workdir_all] = batch_options[:contain]
     end
 
-    IndiferentHash.add_defaults batch_options, 
+    IndiferentHash.add_defaults batch_options,
       :batch_name => batch_name,
-      :inputs_dir => inputs_dir, 
-      :nodes => 1, 
+      :inputs_dir => inputs_dir,
+      :nodes => 1,
       :step_path => job.path,
       :task_cpus => 1,
-      :time => '2min', 
+      :time => '2min',
       :env => {'JDK_JAVA_OPTIONS' => "-Xms1g -Xmx${MAX_MEMORY}m"},
       :singularity_img => ENV["SINGULARITY_IMG"] || "~/scout.singularity.img",
       :singularity_ruby_inline => ENV["SINGULARITY_RUBY_INLINE"] || "~/.singularity_ruby_inline",
       :singularity_opt_dir => ENV["SINGULARITY_OPT_DIR"] || "~/singularity_opt",
-      :workdir => Dir.pwd 
+      :workdir => Dir.pwd
 
     exec_cmd = exec_cmd(job, batch_options)
     scout_cmd = scout_job_exec_cmd(job, options)
 
-    IndiferentHash.add_defaults batch_options, 
+    IndiferentHash.add_defaults batch_options,
       :exec_cmd => exec_cmd,
       :scout_cmd => scout_cmd
 
@@ -344,7 +342,7 @@ batch_erase_contain_dir()
 function batch_sync_contain_dir(){
   mkdir -p "$(dirname '#{sync}')"
   rsync -avztAXHP --copy-unsafe-links "#{source}/" "#{sync}/" 2>1 >> '#{options[:fsync]}'
-  sync_es="$?" 
+  sync_es="$?"
   echo $sync_es > '#{options[:fsexit]}'
   find '#{sync}' -type l -ls | awk '$13 ~ /^#{sync.gsub('/','\/')}/ { sub("#{source}", "#{sync}", $13); print $11, $13 }' | while read A B; do rm $A; ln -s $B $A; done
 }
@@ -353,7 +351,7 @@ function batch_sync_contain_dir(){
 
     if options[:env]
       prepare_environment +=<<-EOF
-# Set ENV variables 
+# Set ENV variables
 #{options[:env].collect{|n,v| "export #{n}=\"#{v}\"" } * "\n"}
       EOF
     end
@@ -384,7 +382,7 @@ for tmpd in persist_locks  produce_locks  R_sockets  sensiblewrite  sensiblewrit
     mkdir -p "#{contain}/.scout/tmp/$tmpd"
 done
 
-# Copy environment 
+# Copy environment
 cp ~/.scout/etc/environment #{contain}/.scout/etc/
 
 # Set search_paths
@@ -406,7 +404,7 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
     exec_cmd, job_cmd, task_cpus = options.values_at :exec_cmd, :scout_cmd, :task_cpus
 
     script=<<-EOF
-step_path=$( 
+step_path=$(
       #{exec_cmd} #{job_cmd} --printpath
 )
 exit_status=$?
@@ -426,7 +424,7 @@ fi
 
     if options[:sync]
       sync_environment +=<<-EOF
-if [ $exit_status == '0' ]; then 
+if [ $exit_status == '0' ]; then
   batch_sync_contain_dir
 else
   sync_es=$exit_status
@@ -441,8 +439,8 @@ fi
     cleanup_environment = ""
 
     cleanup_environment +=<<-EOF if options[:purge_deps]
-if [ $exit_status == '0' ]; then 
-  #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}' 
+if [ $exit_status == '0' ]; then
+  #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}'
 fi
     EOF
 
@@ -453,7 +451,7 @@ batch_erase_contain_dir
         EOF
       elsif options[:wipe_container] == 'post' || options[:wipe_container] == 'both'
         cleanup_environment +=<<-EOF
-if [ $sync_es == '0' -a $empty_contain_dir == 'true' ]; then 
+if [ $sync_es == '0' -a $empty_contain_dir == 'true' ]; then
   batch_erase_contain_dir
 fi
         EOF
@@ -512,7 +510,7 @@ exit $exit_status
 env > #{batch_options[:fenv]}
 
 # #{Log.color :green, "2. Execute"}
-#{execute} 
+#{execute}
 
 # #{Log.color :green, "3. Sync and cleanup environment"}
 #{sync_environment}
@@ -555,13 +553,13 @@ env > #{batch_options[:fenv]}
   def run_job(job, options = {})
     system = self.to_s.split("::").last
 
-    batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run, orchestration_rules_file = IndiferentHash.process_options options, 
+    batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run, orchestration_rules_file = IndiferentHash.process_options options,
       :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run, :orchestration_rules,
       :batch_base_dir => SchedulerJob.batch_base_dir
 
     if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
       Log.info "Job #{job.short_path} already queued in #{batch_job}"
-      return batch_job, batch_dir_for_id(batch_base_dir, batch_job) 
+      return batch_job, batch_dir_for_id(batch_base_dir, batch_job)
     end
 
     if job.running?
@@ -570,7 +568,7 @@ env > #{batch_options[:fenv]}
       if job.info[:batch_job]
         return job.info[:batch_job], batch_dir_for_id(batch_base_dir, batch_job)
       else
-        return 
+        return
       end
     end
 
@@ -582,8 +580,8 @@ env > #{batch_options[:fenv]}
     workflows_to_load = job.rec_dependencies.select{|d| Step === d}.collect{|d| d.workflow }.compact.collect(&:to_s) - [workflow.to_s]
 
     TmpFile.with_file(nil, remove_batch_dir, :tmpdir => batch_base_dir, :prefix => "#{system}_scout_job-#{workflow.to_s}-#{task_name}-") do |batch_dir|
-      IndiferentHash.add_defaults options, 
-        :batch_dir => batch_dir, 
+      IndiferentHash.add_defaults options,
+        :batch_dir => batch_dir,
         :inputs_dir => File.join(batch_dir, "inputs_dir"),
         :workflows => workflows_to_load.any? ? workflows_to_load.uniq * "," : nil