scout-gear 7.3.0 → 8.0.0

data/lib/scout/workflow/step/provenance.rb

@@ -5,7 +5,7 @@ class Step
 
   def self.status_color(status)
     case status.to_sym
-    when :error, :aborted, :missing, :dead, :unsync
+    when :error, :aborted, :dead, :unsync
       :red
     when :streaming, :started
       :cyan
@@ -13,7 +13,7 @@ class Step
       :green
     when :dependencies, :waiting, :setup
       :yellow
-    when :notfound, :cleaned
+    when :notfound, :cleaned, :missing
       :blue
     else
       if status.to_s.index ">"
@@ -91,6 +91,7 @@ class Step
     info[:task_name] = task
     path  = step.path
     status = info[:status] || :missing
+    status = :noinfo if status == :missing && Open.exist?(path)
     status = "remote" if Open.remote?(path) || Open.ssh?(path)
     name = info[:name] || File.basename(path)
     status = :unsync if status == :done and not Open.exist?(path)
@@ -103,9 +104,9 @@ class Step
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
-          input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+          input_name, _dep = dep.recursive_inputs.select{|f,d| 
             d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
-          }.last
+          }.keys.last
           if input_name
             input_dependencies[id] ||= []
             input_dependencies[id] << [dep, input_name]
@@ -115,10 +116,10 @@ class Step
     end if step.dependencies
 
     str = ""
-    str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    str = " " * offset + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
     step.dependencies.dup.tap{|l| 
-      l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+      l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
     }.each do |dep|
       path = dep.path
       new = ! seen.include?(path)
@@ -141,7 +142,7 @@ class Step
       end
     end if step.dependencies
 
-    str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    str += (" " * offset) + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
     str
   end

data/lib/scout/workflow/step/status.rb

@@ -0,0 +1,45 @@
+class Step
+  def abort(exception = nil)
+    while @result && streaming? && stream = self.stream
+      stream.abort(exception)
+    end
+  end
+
+  def recoverable_error?
+    self.error? && ! (ScoutException === self.exception)
+  end
+
+  def updated?
+    return false if self.error? && self.recoverable_error?
+    return true unless ENV["SCOUT_UPDATE"]
+    newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
+    newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
+
+    newer.empty?
+  end
+
+  def clean
+    @take_stream = nil 
+    @result = nil
+    @info = nil
+    @info_load_time = nil
+    Open.rm path if Open.exist?(path)
+    Open.rm info_file if Open.exist?(info_file)
+    Open.rm_rf files_dir if Open.exist?(files_dir)
+  end
+
+  def present?
+    Open.exist?(path) &&
+      Open.exist?(info_file) &&
+      Open.exist?(files_dir)
+  end
+
+
+  def recursive_clean
+    dependencies.each do |dep|
+      dep.recursive_clean
+    end
+    clean
+  end
+
+end

data/lib/scout/workflow/step.rb

@@ -1,6 +1,7 @@
 require_relative '../path'
 require_relative '../persist'
 require_relative 'step/info'
+require_relative 'step/status'
 require_relative 'step/load'
 require_relative 'step/file'
 require_relative 'step/dependencies'
@@ -10,11 +11,13 @@ require_relative 'step/progress'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
-  def initialize(path, inputs = nil, dependencies = nil, &task) 
+  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, &task) 
     @path = path
     @inputs = inputs
     @dependencies = dependencies
+    @id = id
+    @non_default_inputs = non_default_inputs
     @task = task
     @mutex = Mutex.new
     @tee_copies = 1
@@ -26,7 +29,7 @@ class Step
 
   def inputs
     @inputs ||= begin
-                  if Open.exists?(info_file)
+                  if info_file && Open.exists?(info_file)
                     info[:inputs]
                   else
                     []
@@ -55,6 +58,13 @@ class Step
     @name ||= File.basename(@path)
   end
 
+  def clean_name
+    return @id if @id
+    return info[:clean_name] if info.include? :clean_name
+    return m[1] if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
+    return name.split(".").first
+  end
+
   def task_name
     @task_name ||= @task.name if @task.respond_to?(:name)
   end
@@ -64,26 +74,65 @@ class Step
   end
 
   def exec
+
+    if inputs 
+      if Task === task
+        types = task.inputs.collect{|name,type| type }
+        new_inputs = inputs.zip(types).collect{|input,info|  
+          type, desc, default, options = info
+          next input unless Step === input
+          input.join if input.streaming?
+          Task.format_input(input.join.path, type, options)
+        }
+      else
+        new_inputs = inputs.collect{|input|  
+          Step === input ? input.load : input
+        }
+      end
+      inputs = new_inputs
+    end
+
     @result = self.instance_exec(*inputs, &task)
   end
 
+  def tmp_path
+    @tmp_path ||= begin
+                    basename = File.basename(@path)
+                    dirname = File.dirname(@path)
+                    tmp_path = File.join(dirname, '.' + basename)
+                    @path.setup(tmp_path) if Path === @path
+                    tmp_path
+                  end
+  end
+
   attr_reader :result
-  def run
+  def run(stream = false)
     return @result || self.load if done?
     prepare_dependencies
     run_dependencies
-    @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
+    @result = 
       begin
-        merge_info :status => :start, :start => Time.now,
-          :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
-          :inputs => inputs, :type => type,
-          :dependencies => dependencies.collect{|d| d.path }
+        Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
+          clear_info
+          merge_info :status => :start, :start => Time.now,
+            :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
+            :inputs => inputs, :type => type,
+            :dependencies => dependencies.collect{|d| d.path }
 
-        @result = exec
-        @result = @result.respond_to?(:stream) ? @result.stream : @result
-        @result
+
+          @result = exec
+
+          if @result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
+            Open.mv self.tmp_path, self.path
+          else
+            @result = @result.stream if @result.respond_to?(:stream)
+          end
+
+          @result
+        end
       rescue Exception => e
-        merge_info :status => :error, :exception => e
+        merge_info :status => :error, :exception => e, :end => Time.now
+        abort_dependencies
         raise e
       ensure
         if ! (error? || aborted?)
@@ -106,8 +155,18 @@ class Step
           end
         end
       end
+
+    if stream && ENV["SCOUT_NO_STREAM"].nil?
+      @result
+    else
+      if IO === @result || @result.respond_to?(:stream)
+        join
+        @result = nil
+        self.load
+      else
+        @result
+      end
     end
-    @result
   end
 
   def done?
@@ -140,9 +199,23 @@ class Step
     end
   end
 
+  def consume_all_streams
+    threads = [] 
+    while @result && streaming? && stream = self.stream
+      threads << Open.consume_stream(stream, true)
+    end
+    threads.each do |t| 
+      begin
+        t.join 
+      rescue
+        threads.each{|t| t.raise(Aborted); t.join }
+        raise $!
+      end
+    end
+  end
+
   def join
-    io = self.stream if streaming?
-    Open.consume_stream(io, false) if io
+    consume_all_streams
     self
   end
 
@@ -157,31 +230,24 @@ class Step
     done? ? Persist.load(path, type) : exec
   end
 
-  def clean
-    @take_stream = nil 
-    @result = nil
-    @info = nil
-    @info_load_time = nil
-    Open.rm path if Open.exist?(path)
-    Open.rm info_file if Open.exist?(info_file)
-    Open.rm_rf files_dir if Open.exist?(files_dir)
-  end
-
-  def recursive_clean
-    dependencies.each do |dep|
-      dep.recursive_clean
-    end
-    clean
-  end
-
   def step(task_name)
     dependencies.each do |dep|
       return dep if dep.task_name == task_name
+      rec_dep = dep.step(task_name)
+      return rec_dep if rec_dep
     end
     nil
   end
 
+  def short_path
+    Scout.identify @path
+  end
+
   def digest_str
-    path.dup
+    "Step: " + short_path
+  end
+
+  def fingerprint
+    digest_str
   end
 end

data/lib/scout/workflow/task/inputs.rb

@@ -1,11 +1,10 @@
+require_relative '../../named_array'
 module Task
+  def self.format_input(value, type, options = {})
+    return value if IO === value || StringIO === value || Step === value
 
-  def format_input(value, type, options = {})
-    return value if IO === value || StringIO === value
-
-    value = value.load if Step === value
-    if String === value && ! [:path, :file].include?(type) 
-      if Open.exists?(value)
+    if String === value && ! [:path, :file, :folder].include?(type) && (options.nil? || (! options[:noload] || options[:stream]))
+      if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else
         Persist.deserialize(value, type)
@@ -13,7 +12,7 @@ module Task
     else
       if m = type.to_s.match(/(.*)_array/)
         if Array === value
-          value.collect{|v| format_input(v, m[1].to_sym, options) }
+          value.collect{|v| self.format_input(v, m[1].to_sym, options) }
         end
       else
         value
@@ -34,35 +33,30 @@ module Task
     IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
 
     input_array = []
+    input_names = []
     non_default_inputs = []
     self.inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
+      input_names << name
       provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
-      provided = format_input(provided, type, options || {})
+      provided = Task.format_input(provided, type, options || {})
       if ! provided.nil? && provided != value
-        non_default_inputs << name
+        non_default_inputs << name.to_sym
         input_array << provided
       else
         input_array << value
       end
     end
 
+    NamedArray.setup(input_array, input_names)
+
     [input_array, non_default_inputs]
   end
 
-  def digest_inputs(provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs(provided_inputs)
-    if Array === provided_inputs 
-      Misc.digest(input_array)
-    else
-      Misc.digest(input_array)
-    end
-  end
-  
   def process_inputs(provided_inputs = {})
     input_array, non_default_inputs = assign_inputs provided_inputs
-    digest = Misc.digest(input_array)
-    [input_array, non_default_inputs, digest]
+    digest_str = Misc.digest_str(input_array)
+    [input_array, non_default_inputs, digest_str]
   end
 
   def save_file_input(orig_file, directory)

data/lib/scout/workflow/task.rb

@@ -44,27 +44,45 @@ module Task
     
     provided_inputs ||= {}
 
-    load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
-      task = hash_options[:task] if hash_options.include?(:task)
-      workflow = hash_options[:workflow] if hash_options.include?(:workflow)
-      id = hash_options[:id] if hash_options.include? :id
+    # Helper function
+    load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
+      task = step_options.delete(:task) if step_options.include?(:task)
+      workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
+      id = step_options.delete(:id) if step_options.include?(:id)
+      id = step_options.delete(:jobname) if step_options.include?(:jobname)
 
-      hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
-      inputs = IndiferentHash.add_defaults hash_inputs, inputs
+      step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
+      step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
 
       resolved_inputs = {}
-      inputs.each do |k,v|
+      step_inputs.each do |k,v|
         if Symbol === v
           input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || inputs[v] || k
+          resolved_inputs[k] = input_dep || step_inputs[v] || k
         else
           resolved_inputs[k] = v
         end
       end
-      workflow.job(task, id, resolved_inputs)
+      [workflow.job(task, id, resolved_inputs), step_inputs]
     end
 
-    deps.each do |workflow,task,options,block=nil|
+    # Helper function
+    find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
+      dep_non_default_inputs = dep.non_default_inputs
+      dep_non_default_inputs.select do |name|
+        step_inputs.include?(name)  
+      end
+      dep_non_default_inputs.reject! do |name|
+        definition_options.include?(name) && 
+          (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
+      end
+
+      dep_non_default_inputs
+    end
+
+    deps.each do |workflow,task,definition_options,block=nil|
+      definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
+
       if provided_inputs.include?(overriden = [workflow.name, task] * "#")
         dep = provided_inputs[overriden]
         dep = Step.new dep unless Step === dep
@@ -74,58 +92,52 @@ module Task
         next
       end
 
-      options ||= {}
+      definition_options ||= {}
+
       if block
-        inputs = IndiferentHash.add_defaults options.dup, provided_inputs
+        fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
+        self.inputs.each do |name,type,desc,value|
+          fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
+        end
+        fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
+        block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
 
-        res = block.call id, inputs, dependencies
+        res = block.call id, block_options, dependencies
 
         case res
         when Step
           dep = res
           dependencies << dep
-          dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
-          non_default_inputs.concat(dep_non_default_inputs - options.keys)
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+          non_default_inputs.concat(dep_non_default_inputs)
         when Hash
-          new_options = res
-          dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
+          step_options = block_options.merge(res)
+          dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
           dependencies << dep
-          dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
-          dep_non_default_inputs -= options.keys 
-          if new_options.include?(:inputs)
-            dep_non_default_inputs -= new_options[:inputs].keys 
-          else
-            dep_non_default_inputs -= new_options.keys
-          end
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
           non_default_inputs.concat(dep_non_default_inputs)
         when Array
           res.each do |_res|
             if Hash === _res
-              new_options = _res
-              dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
+              step_options = block_options.merge(_res)
+              dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
               dependencies << dep
-              dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
-              dep_non_default_inputs -= options.keys 
-              if new_options.include?(:inputs)
-                dep_non_default_inputs -= new_options[:inputs].keys 
-              else
-                dep_non_default_inputs -= new_options.keys
-              end
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
               non_default_inputs.concat(dep_non_default_inputs)
             else
               dep = _res
               dependencies << dep
-              dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
-              non_default_inputs.concat(dep_non_default_inputs - options.keys)
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+              non_default_inputs.concat(dep_non_default_inputs)
             end
           end
         end
       else
-        inputs = IndiferentHash.add_defaults options.dup, provided_inputs
-        dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
+        step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
+        dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
         dependencies << dep
-        dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
-        non_default_inputs.concat(dep_non_default_inputs - options.keys)
+        dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+        non_default_inputs.concat(dep_non_default_inputs)
       end
     end
 
@@ -137,24 +149,47 @@ module Task
     provided_inputs = {} if provided_inputs.nil?
     id = DEFAULT_NAME if id.nil?
 
-    inputs, non_default_inputs, input_hash = process_inputs provided_inputs
+    inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
 
     dependencies = dependencies(id, provided_inputs, non_default_inputs)
 
     non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
     if non_default_inputs.any?
-      hash = Misc.digest(:inputs => input_hash, :dependencies => dependencies)
-      Log.debug "Hash #{name} - #{hash}: #{Log.fingerprint(:inputs => inputs, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
-      id = [id, hash] * "_"
+      hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
+      name = [id, hash] * "_"
+    else
+      name = id
     end
 
-    path = directory[id]
+    extension = self.extension
+    if extension == :dep_task
+      extension = nil
+      if dependencies.any?
+        dep_basename = File.basename(dependencies.last.path)
+        if dep_basename.include? "."
+          parts = dep_basename.split(".")
+          extension = [parts.pop]
+          while parts.last.length <= 4
+            extension << parts.pop
+          end
+          extension = extension.reverse * "."
+        end
+      end
+    end
+
+    path = directory[name]
+
+    path = path.set_extension(extension) if extension
 
     Persist.memory(path) do 
-      Log.debug "Creating job #{path} #{Log.fingerprint inputs} #{Log.fingerprint dependencies}"
+      if hash
+        Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
+      else
+        Log.debug "ID #{self.name} #{id} - Clean"
+      end
       NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
-      Step.new path.find, inputs, dependencies, &self
+      Step.new path.find, inputs, dependencies, id, non_default_inputs, &self
     end
   end
 end

data/lib/scout/workflow/usage.rb

@@ -16,14 +16,16 @@ module Task
       str.puts
     end
 
+    deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
     if deps and deps.any?
       seen = inputs.collect{|name,_| name }
       dep_inputs = {}
-      deps.each do |dep_workflow,task_name|
+      deps.each do |dep_workflow,task_name,options|
         next if task_name.nil?
         task = dep_workflow.tasks[task_name]
         next if task.inputs.nil?
         inputs = task.inputs.reject{|name, _| seen.include? name }
+        inputs = task.inputs.reject{|name, _| options.include? name }
         next unless inputs.any?
         dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
         dep_inputs[dep] = inputs
@@ -109,25 +111,25 @@ end
 
 module Workflow
 
-  def dep_tree(task_name, seen = nil)
+  def dep_tree(task_name, seen = nil, seen_options = nil)
     @dep_tree ||= {}
     key = [self, task_name]
 
     return @dep_tree[key] if @dep_tree.include?(key)
     save = seen.nil?
     seen = Set.new if seen.nil?
+    seen_options = {} if seen_options.nil?
 
     dep_tree = {}
     task = self.tasks[task_name]
-    task.deps.each do |dep|
+    task.deps.each do |workflow, task, options|
       next if seen.include? dep
-      seen << dep
-      workflow, task, *rest = dep
+      seen << [workflow, task, options.merge(seen_options)]
       next if task.nil?
 
       key = [workflow, task]
 
-      dep_tree[key] = workflow.dep_tree(task, seen)
+      dep_tree[key] = workflow.dep_tree(task, seen, options.merge(seen_options))
     end if task.deps
 
     @dep_tree[key] = dep_tree if save