scout-gear 10.11.4 → 10.11.7

data/lib/scout/workflow/deployment/scheduler/lfs.rb

@@ -1,7 +1,7 @@
 require_relative 'job'
 require 'scout'
 
-module LSF 
+module LSF
   extend SchedulerJob
 
   def self.system

data/lib/scout/workflow/deployment/scheduler/pbs.rb

@@ -1,7 +1,7 @@
 require_relative 'job'
 require 'scout'
 
-module PBS 
+module PBS
   extend SchedulerJob
 
   def self.system
@@ -13,7 +13,7 @@ module PBS
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
 let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $PBS_CPUS_PER_TASK )"
 MAX_MEMORY="$MAX_MEMORY_DEFAULT"
-[ ! -z $PBS_MEM_PER_CPU ] && let MAX_MEMORY="$PBS_MEM_PER_CPU * $PBS_CPUS_PER_TASK" 
+[ ! -z $PBS_MEM_PER_CPU ] && let MAX_MEMORY="$PBS_MEM_PER_CPU * $PBS_CPUS_PER_TASK"
 [ ! -z $PBS_MEM_PER_NODE ] && MAX_MEMORY="$PBS_MEM_PER_NODE"
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY
@@ -36,7 +36,7 @@ cd ${PBS_O_WORKDIR}
     time       = IndiferentHash.process_options options, :time
     nodes      = IndiferentHash.process_options options, :nodes
 
-    # PBS 
+    # PBS
     place      = IndiferentHash.process_options options, :place, :place => 'scatter'
     system     = IndiferentHash.process_options options, :partition
     filesystems = IndiferentHash.process_options options, :filesystems
@@ -45,7 +45,7 @@ cd ${PBS_O_WORKDIR}
 
     filesystems = filesystems * "," if Array === filesystems
 
-    # NOT USED 
+    # NOT USED
     partition  = IndiferentHash.process_options options, :partition
     task_cpus  = IndiferentHash.process_options options, :task_cpus
     exclusive  = IndiferentHash.process_options options, :exclusive

data/lib/scout/workflow/deployment/scheduler/slurm.rb

@@ -1,7 +1,7 @@
 require_relative 'job'
 require 'scout'
 
-module SLURM 
+module SLURM
 
   extend SchedulerJob
 
@@ -15,7 +15,7 @@ module SLURM
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
 let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $SLURM_CPUS_PER_TASK )"
 MAX_MEMORY="$MAX_MEMORY_DEFAULT"
-[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" 
+[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK"
 [ ! -z $SLURM_MEM_PER_NODE ] && MAX_MEMORY="$SLURM_MEM_PER_NODE"
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY

data/lib/scout/workflow/deployment/scheduler.rb

@@ -12,31 +12,42 @@ module Workflow::Scheduler
     Workflow::Scheduler.process_batches(batches, options)
   end
 
+  def self.produce_single_batch(job, rules = {}, options = {})
+    batches = Workflow::Orchestrator.job_batches(rules, job)
+    rules = batches.inject({}){|acc,b| acc = Workflow::Orchestrator.accumulate_rules acc, b[:rules] }
+
+    max_time = batches.collect{|b| b[:rules][:time] }.max
+    rules[:time] = options.include?(:time) ? options[:time] : max_time
+    rules[:deploy] = :local
+    if rules[:exclusive]
+      rules[:task_cpus] = nil
+      rules[:cpus] = nil
+    end
+
+    batch = {top_level: job, jobs: [job], rules: rules, deps: [] }
+    Workflow::Scheduler.process_batches([batch], options)
+  end
+
+
   def self.process_batches(batches, process_options = {})
     failed_jobs = []
 
-    pending = batches.dup
-
-    sorted = []
-    while pending.any?
-      leaf_nodes = batches.select{|batch| (batch[:deps] - sorted).empty? }
-      sorted.concat(leaf_nodes - sorted)
-      pending -= leaf_nodes
-    end
+    sorted = Workflow::Orchestrator.sort_batches batches
+    cleaned = Workflow::Orchestrator.clean_batches sorted
 
     batch_system = Scout::Config.get :system, :batch, :scheduler, 'env:BATCH_SYSTEM', default: 'SLURM'
 
     batch_ids = {}
-    sorted.collect do |batch|
+    cleaned.collect do |batch|
       job_options = batch[:rules]
       job_options = IndiferentHash.add_defaults job_options, process_options.dup
 
       if batch[:deps].nil?
-        batch_dependencies = [] 
-      else 
+        batch_dependencies = []
+      else
         top_jobs = batch[:jobs]
 
-        batch_dependencies = batch[:deps].collect{|dep| 
+        batch_dependencies = batch[:deps].collect{|dep|
           dep_target = dep[:top_level]
           id = batch_ids[dep_target].to_s
 

data/lib/scout/workflow/deployment/trace.rb

@@ -132,10 +132,10 @@ module Workflow
       info[:time] << time
 
       report_keys.each do |key|
-        info[key] = dep_info[key] 
+        info[key] = dep_info[key]
       end
 
-      dep.info[:config_keys].each do |kinfo| 
+      dep.info[:config_keys].each do |kinfo|
         key, value, tokens = kinfo
 
         info[key.to_s] = value if report_keys.include? key.to_s

data/lib/scout/workflow/documentation.rb

@@ -31,7 +31,7 @@ module Workflow
     title = doc_parse_first_line doc
     description, task_info = doc_parse_up_to doc, /^# Tasks/i
     task_description, tasks = doc_parse_up_to task_info, /^##/, true
-    tasks = doc_parse_chunks tasks, /^## (.*)/ 
+    tasks = doc_parse_chunks tasks, /^## (.*)/
     {:title => title.strip, :description => description.strip, :task_description => task_description.strip, :tasks => tasks}
   end
 
@@ -52,11 +52,11 @@ module Workflow
                          documentation = Workflow.parse_workflow_doc documentation_markdown
 
                          if @description && (documentation[:description].nil? || documentation[:description].empty?)
-                           documentation[:description] = @description 
+                           documentation[:description] = @description
                          end
 
                          if @title && (documentation[:title].nil? || documentation[:title].empty?)
-                           documentation[:title] = @title 
+                           documentation[:title] = @title
                          end
                          documentation[:tasks].each do |task, description|
                            if task.include? "#"
@@ -74,7 +74,7 @@ module Workflow
                            if workflow.tasks.include? task
                              workflow.tasks[task].description = description
                            else
-                             Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
+                             Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task"
                            end
                          end
                          documentation

data/lib/scout/workflow/export.rb

@@ -39,7 +39,7 @@ module Workflow
     asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
     stream_exports.replace stream_exports - names if stream_exports
   end
-  
+
   def export_exec(*names)
     unexport *names
     exec_exports.concat names

data/lib/scout/workflow/path.rb

@@ -11,9 +11,9 @@ module Path
         _loaded = false
         begin
           Kernel.const_get(workflow)
-        rescue 
+        rescue
           if ! _loaded
-            Workflow.require_workflow workflow 
+            Workflow.require_workflow workflow
             _loaded = true
             retry
           end

data/lib/scout/workflow/step/children.rb

@@ -40,7 +40,7 @@ class Step
       if c == "\n"
         Log.logn "STDOUT [#{child_pid}]: ", level
       end
-    end 
+    end
 
     io.join
 

data/lib/scout/workflow/step/dependencies.rb

@@ -1,4 +1,5 @@
 class Step
+
   def rec_dependencies(connected = false, seen = Set.new)
     @rec_dependencies = {}
     @rec_dependencies[connected] ||= begin
@@ -34,6 +35,31 @@ class Step
     end.compact.uniq
   end
 
+  def overrider?
+    ! (overriden_task.nil? && overriden_workflow.nil?)
+  end
+
+  def overriden?
+    @overriden
+  end
+
+  def overriden_deps
+    dependencies.select{|d| d.overriden? }
+  end
+
+  def recursive_overriden_deps
+    overriden_deps = self.overriden_deps
+    overriden_deps + overriden_deps.collect{|dep| dep.recursive_overriden_deps }
+  end
+
+  def overrider_deps
+    dependencies.select{|d| d.overrider? }
+  end
+
+  def recursive_overrider_deps
+    self.rec_dependencies.select{|dep| dep.overrider? }
+  end
+
   def prepare_dependencies
     inverse_dep = {}
 
@@ -44,7 +70,14 @@ class Step
       end
 
       next if dep.done?
-      next if dep.error? && ! dep.recoverable_error?
+
+      if dep.error? && ! dep.recoverable_error?
+        if dep.canfail?
+          next
+        else
+          raise dep.exception
+        end
+      end
 
       if dep.dependencies
         dep.dependencies.each do |d|
@@ -74,7 +107,7 @@ class Step
   end
 
   def run_dependencies
-    all_dependencies.each do |dep| 
+    all_dependencies.each do |dep|
       next if dep.running? || dep.done?
       next if dep.error? && ! dep.recoverable_error?
 
@@ -100,7 +133,7 @@ class Step
   end
 
   def abort_dependencies
-    all_dependencies.each{|dep| dep.abort if dep.running? } 
+    all_dependencies.each{|dep| dep.abort if dep.running? }
   end
 
   def self.wait_for_jobs(jobs, canfail=false)

data/lib/scout/workflow/step/info.rb

@@ -30,7 +30,8 @@ class Step
   end
 
   def save_info(info = nil)
-    Persist.save(@info = info, info_file, SERIALIZER)
+    purged = Annotation.purge(@info = info)
+    Persist.save(purged, info_file, SERIALIZER)
     @info_load_time = Time.now
   end
 
@@ -82,6 +83,7 @@ class Step
 
           if start && eend
             time = eend - start
+            Log.warn "No issue time #{self.path}" if issued.nil?
             total_time = eend - issued
             if total_time - time > 1
               time_str = "#{Misc.format_seconds_short(time)} (#{Misc.format_seconds_short(total_time)})"
@@ -93,7 +95,7 @@ class Step
             message = Log.color(:time, time_str)
           end
         end
-        report_status value, message 
+        report_status value, message
       end
 
       if key == :message
@@ -136,7 +138,7 @@ class Step
   def set_info(key, value)
     merge_info(key => value)
   end
-  
+
   def report_status(status, message = nil)
     if message.nil?
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
@@ -184,22 +186,6 @@ class Step
     ! (done? && status == :done) && (info[:pid] && Misc.pid_alive?(info[:pid]))
   end
 
-  def overriden?
-    @overriden = overriden_task || overriden_workflow || overriden_deps.any? if @overriden.nil?
-    @overriden
-  end
-
-  def overriden_deps
-    dependencies.select{|d| d.overriden? }
-  end
-
-  def recursive_overriden_deps
-    overriden_deps = dependencies.select{|d| 
-      d.overriden? 
-    }
-    (overriden_deps + overriden_deps.collect{|d| d.recursive_overriden_deps }).flatten.uniq
-  end
-
   def exception
     return nil unless info[:exception]
     begin

data/lib/scout/workflow/step/inputs.rb

@@ -3,7 +3,7 @@ class Step
     if clean_name != name
       #hash = name[clean_name.length..-1]
       #inputs_dir += hash
-      Log.medium "Saving job inputs to: #{Log.fingerprint inputs_dir}"
+      Log.medium "Saving job inputs to: #{Log.fingerprint inputs_dir} #{Log.fingerprint provided_inputs}"
       self.task.save_inputs(inputs_dir, provided_inputs)
     else
       Log.medium "Saving no input job: #{Log.fingerprint inputs_dir}"

data/lib/scout/workflow/step/progress.rb

@@ -27,9 +27,9 @@ class Step
   end
 
   def monitor_stream(stream, options = {}, &block)
-    case options[:bar] 
+    case options[:bar]
     when TrueClass
-      bar = progress_bar 
+      bar = progress_bar
     when Hash
       bar = progress_bar options[:bar]
     when Numeric

data/lib/scout/workflow/step/provenance.rb

@@ -58,10 +58,10 @@ class Step
     str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
             prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
           else
-            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " 
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " "
           end
 
-    if $inputs and $inputs.any? 
+    if $inputs and $inputs.any?
       job_inputs = Step.new(path).recursive_inputs.to_hash
       IndiferentHash.setup(job_inputs)
 
@@ -119,7 +119,7 @@ class Step
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
-          input_name, _dep = dep.recursive_inputs.select{|f,d| 
+          input_name, _dep = dep.recursive_inputs.select{|f,d|
             d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
           }.keys.last
           if input_name
@@ -135,7 +135,7 @@ class Step
     indent = prov_indent(step, offset, input_dependencies)
     str << indent + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
-    step.dependencies.dup.tap{|l| 
+    step.dependencies.dup.tap{|l|
       l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
     }.each do |dep|
       path = dep.path

data/lib/scout/workflow/step/status.rb

@@ -5,7 +5,7 @@ class Step
       begin
         s = Misc.abort_child pid, true
         Log.medium "Aborted pid #{path} #{s}"
-      rescue 
+      rescue
         Log.debug("Aborted job #{pid} was not killed: #{$!.message}")
       end
     else
@@ -17,35 +17,49 @@ class Step
   end
 
   def recoverable_error?
-    self.error? && ! (ScoutException === self.exception)
+    self.error? && ! (ENV['SCOUT_NO_RECOVERABLE_ERROR'].to_s.downcase == 'true') && ! (ScoutException === self.exception)
   end
 
   def newer_dependencies
+    rec_dependencies = self.rec_dependencies
+    rec_dependencies = rec_dependencies.select{|dep| Open.exists?(dep.info_file) }
+    rec_dependencies = rec_dependencies.reject{|dep| dep.error? && ! dep.recoverable_error? }
     newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| Path.newer?(self.path, dep.path) }
     newer
   end
 
+  def cleaned_dependencies
+    return []
+    rec_dependencies = self.rec_dependencies
+    cleaned = rec_dependencies.select{|dep| dep.info[:status] == :cleaned }
+    cleaned += input_dependencies.select{|dep| dep.info[:status] == :cleaned }
+    cleaned += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| dep.info[:status] == :cleaned }
+    cleaned
+  end
+
   def updated?
     return false if self.error? && self.recoverable_error?
-    return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"]
+    return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"].to_s.downcase == 'true'
     newer = newer_dependencies
+    cleaned = cleaned_dependencies
 
     Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
-    newer.empty?
+    Log.low "Cleaned deps found for #{Log.fingerprint self}: #{Log.fingerprint cleaned}" if cleaned.any?
+    newer.empty? && cleaned.empty?
   end
 
   def clean
     Log.debug "Cleaning job files: #{path}"
-    @take_stream = nil 
+    @take_stream = nil
     @result = nil
     @info = nil
     @info_load_time = nil
     @done = nil
-    Open.rm path if Open.exist_or_link?(path)
-    Open.rm tmp_path if Open.exist_or_link?(tmp_path)
-    Open.rm info_file if Open.exist_or_link?(info_file)
+    Open.rm path if Open.exist_or_link?(path) && Path.can_write?(path)
+    Open.rm tmp_path if Open.exist_or_link?(tmp_path) && Path.can_write?(tmp_path)
+    Open.rm info_file if Open.exist_or_link?(info_file) && Path.can_write?(info_file)
     Open.rm_rf files_dir if Open.exist_or_link?(files_dir)
     self
   end
@@ -59,7 +73,7 @@ class Step
     dependencies.each do |dep|
       dep.recursive_clean
     end
-    clean
+    clean if Open.exists?(self.info_file)
   end
 
   def canfail?

data/lib/scout/workflow/step.rb

@@ -13,17 +13,18 @@ require_relative 'step/inputs'
 require_relative 'step/children'
 require_relative 'step/archive'
 
-class Step 
+class Step
 
   attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context: nil, &task)
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, overriden = nil, exec_context: nil, &task)
     @path = path
     @inputs = inputs
     @dependencies = dependencies
     @id = id
     @non_default_inputs = non_default_inputs
     @provided_inputs = provided_inputs
-    @compute = compute 
+    @compute = compute
+    @overriden = overriden unless overriden.nil?
     @task = task
     @mutex = Mutex.new
     @tee_copies = 1
@@ -36,12 +37,12 @@ class Step
 
   def provided_inputs
     @provided_inputs ||= begin
-                  if info_file && Open.exists?(info_file)
-                    info[:provided_inputs]
-                  else
-                    {}
-                  end
-                end
+                           if info_file && Open.exists?(info_file)
+                             info[:provided_inputs]
+                           else
+                             {}
+                           end
+                         end
   end
 
   def non_default_inputs
@@ -97,7 +98,7 @@ class Step
     return @id if @id
     return info[:clean_name] if info.include? :clean_name
     if m = name.match(/(.+?)(?:_[a-z0-9]{32})?(?:\..*)?$/)
-      return m[1] 
+      return m[1]
     end
     return name.split(".").first
   end
@@ -189,9 +190,10 @@ class Step
       end
     end
 
-
     return @result || self.load if done?
+
     prepare_dependencies
+
     begin
 
 
@@ -200,11 +202,11 @@ class Step
 
 
         reset_info :status => :setup, :issued => Time.now,
-          :pid => Process.pid, :pid_hostname => Misc.hostname, 
+          :pid => Process.pid, :pid_hostname => Misc.hostname,
           :task_name => task_name, :workflow => workflow.to_s,
-          :provided_inputs => Annotation.purge(provided_inputs),
+          :provided_inputs => IndiferentHash.serializable(provided_inputs),
           :non_default_inputs => non_default_inputs,
-          :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
+          :inputs => IndiferentHash.serializable(inputs), :input_names => input_names, :type => type,
           :dependencies => (dependencies || []) .collect{|d| d.path }
 
         run_dependencies
@@ -313,7 +315,7 @@ class Step
   end
 
   def streaming?
-    @take_stream || IO === @result || StringIO === @result 
+    @take_stream || IO === @result || StringIO === @result
   end
 
   def stream
@@ -344,16 +346,16 @@ class Step
   end
 
   def consume_all_streams
-    threads = [] 
+    threads = []
     while @result && streaming? && stream = self.stream
       threads << Open.consume_stream(stream, true)
     end
 
     threads.compact!
 
-    threads.each do |t| 
+    threads.each do |t|
       begin
-        t.join 
+        t.join
       rescue Exception
         threads.compact.each{|t| t.raise(Aborted); t.join }
         raise $!
@@ -388,7 +390,7 @@ class Step
 
     raise self.exception if self.exception
 
-    raise "Error in job #{self.path}" if self.error? or self.aborted? 
+    raise "Error in job #{self.path}" if self.error? or self.aborted?
 
     self
   end

data/lib/scout/workflow/task/dependencies.rb

@@ -2,7 +2,7 @@ module Task
   def dependencies(id, provided_inputs, non_default_inputs = [], compute = {})
     return [] if deps.nil?
     dependencies = []
-    
+
     provided_inputs ||= {}
 
     # Helper function
@@ -66,8 +66,15 @@ module Task
 
       if provided_inputs.include?(overriden = [workflow.name, task] * "#")
         dep = provided_inputs[overriden]
-        dep = Step.new dep unless Step === dep
-        dep = dep.dup
+        dep = case dep
+              when Step
+                dep
+              when String
+                Step.new dep
+              when Symbol
+                dependencies.select{|dep| dep.task_name == dep }.last
+              end
+
         dep.type = workflow.tasks[task].type
         dep.overriden_task = task
         dep.overriden_workflow = workflow

data/lib/scout/workflow/task/info.rb

@@ -42,8 +42,8 @@ module Workflow
   def rec_input_use(task_name)
     input_use = {}
     task = self.tasks[task_name]
-    task.inputs.each do |name,_| 
-      input_use[name] ||= {} 
+    task.inputs.each do |name,_|
+      input_use[name] ||= {}
       input_use[name][self] ||= []
       input_use[name][self] << task_name
     end
@@ -69,7 +69,7 @@ module Workflow
     raise "No '#{name}' task in '#{self.name}' Workflow" if task.nil?
     id = File.join(self.name, name.to_s)
     @task_info ||= {}
-    @task_info[id] ||= begin 
+    @task_info[id] ||= begin
                          description = task.description
                          returns = task.returns
 

data/lib/scout/workflow/task/inputs.rb

@@ -3,9 +3,9 @@ module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value
 
-    if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
+    if String === value && Path.is_filename?(value) && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
       if Open.exists?(value) && ! Open.directory?(value)
-        Persist.load(value, type) 
+        Persist.load(value, type)
       else
         Persist.deserialize(value, type)
       end
@@ -96,7 +96,7 @@ module Task
       basename += "-#{digest}"
     end
     new_file = File.join(directory, 'saved_input_files', basename)
-    relative_file = File.join('.', 'saved_input_files', basename) 
+    relative_file = File.join('.', 'saved_input_files', basename)
     Open.link orig_file, new_file
     relative_file
   end
@@ -104,7 +104,7 @@ module Task
   def self.save_input(directory, name, type, value)
     input_file = File.join(directory, name.to_s)
 
-    if Path.is_filename?(value) 
+    if Path.is_filename?(value)
       if type == :path
         Open.write(input_file + ".as_path", value)
       elsif Path.step_file?(value)
@@ -114,7 +114,7 @@ module Task
         Open.write(input_file + ".as_file", relative_file)
       end
     elsif Step === value
-      Open.write(input_file + ".as_step", value.short_path)
+      Open.write(input_file + ".as_step", value.identify_path)
     elsif type == :file
       relative_file = save_file_input(value, directory)
       Persist.save(relative_file, input_file, :file)
@@ -153,8 +153,8 @@ module Task
         value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
         value
       elsif filename.end_with?('.as_step')
-        value = Open.read(filename).strip
-        Step.load value
+        value = Path.setup Open.read(filename).strip
+        Step.load value.find
       elsif filename.end_with?('.as_path')
         value = Open.read(filename).strip
         Path.setup value
@@ -183,7 +183,7 @@ module Task
     self.recursive_inputs.each do |p|
       name, type, desc, value, options = p
       next if seen.include?(name)
-      filename = File.join(directory, name.to_s) 
+      filename = File.join(directory, name.to_s)
       value = Task.load_input_from_file(filename, type, options)
       inputs[name] = value unless value.nil?
       seen << name