scout-gear 8.0.0 → 9.0.0

data/lib/scout/workflow/step.rb

@@ -8,16 +8,19 @@ require_relative 'step/dependencies'
 require_relative 'step/provenance'
 require_relative 'step/config'
 require_relative 'step/progress'
+require_relative 'step/inputs'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, &task) 
+  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
     @path = path
     @inputs = inputs
     @dependencies = dependencies
     @id = id
     @non_default_inputs = non_default_inputs
+    @provided_inputs = provided_inputs
+    @compute = compute 
     @task = task
     @mutex = Mutex.new
     @tee_copies = 1
@@ -42,7 +45,7 @@ class Step
                         if Open.exists?(info_file)
                           info[:dependencies].collect do |path|
                             Step.load(path)
-                          end
+                          end if info[:dependencies]
                         else
                           []
                         end
@@ -92,7 +95,12 @@ class Step
       inputs = new_inputs
     end
 
-    @result = self.instance_exec(*inputs, &task)
+    @result = begin
+                @in_exec = true
+                self.instance_exec(*inputs, &task)
+              ensure
+                @in_exec = false
+              end
   end
 
   def tmp_path
@@ -115,11 +123,10 @@ class Step
         Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
           clear_info
           merge_info :status => :start, :start => Time.now,
-            :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
-            :inputs => inputs, :type => type,
+            :pid => Process.pid, :pid_hostname => Misc.hostname, 
+            :inputs => MetaExtension.purge(inputs), :type => type,
             :dependencies => dependencies.collect{|d| d.path }
 
-
           @result = exec
 
           if @result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
@@ -129,6 +136,13 @@ class Step
           end
 
           @result
+
+          if (IO === @result || StringIO === @result) && (ENV["SCOUT_NO_STREAM"] == "true" || ! stream)
+            Open.sensible_write(self.path, @result)
+            @result = nil
+          else
+            @result
+          end
         end
       rescue Exception => e
         merge_info :status => :error, :exception => e, :end => Time.now
@@ -142,10 +156,14 @@ class Step
             end
 
             @result.abort_callback = proc do |exception|
-              if Aborted === exception || Interrupt === exception
+              if exception.nil? || Aborted === exception || Interrupt === exception
                 merge_info :status => :aborted, :end => Time.now
               else
-                merge_info :status => :error, :exception => exception, :end => Time.now
+                begin
+                  merge_info :status => :error, :exception => exception, :end => Time.now
+                rescue Exception
+                  Log.exception $!
+                end
               end
             end
 
@@ -155,18 +173,16 @@ class Step
           end
         end
       end
+  end
 
-    if stream && ENV["SCOUT_NO_STREAM"].nil?
-      @result
-    else
-      if IO === @result || @result.respond_to?(:stream)
-        join
-        @result = nil
-        self.load
-      else
-        @result
-      end
+  def fork
+    Process.fork do
+      clear_info unless present?
+      run(false)
+      join
     end
+    grace
+    self
   end
 
   def done?
@@ -185,16 +201,21 @@ class Step
         else
           Log.debug "Taking result #{Log.fingerprint @result}"
         end
+
         @take_stream, @result = @result, @result.next
-        @take_stream
-      elsif done?
+
+        return @take_stream
+      end
+    end
+
+    if done?
+      Open.open(self.path)
+    else
+      if running? || waiting?
+        join
         Open.open(self.path)
       else
-        if running?
-          nil
-        else
-          exec
-        end
+        exec
       end
     end
   end
@@ -207,21 +228,47 @@ class Step
     threads.each do |t| 
       begin
         t.join 
-      rescue
+      rescue Exception
         threads.each{|t| t.raise(Aborted); t.join }
         raise $!
       end
     end
   end
 
+  def present?
+    Open.exist?(path) ||
+      Open.exist?(info_file) ||
+      Open.exist?(files_dir)
+  end
+
+  def grace
+    while ! present?
+      sleep 0.1
+    end
+  end
+
+  def terminated?
+    ! @in_exec && (done? || error? || aborted?)
+  end
+
   def join
     consume_all_streams
+    while @result.nil? && (present? && ! (terminated? || done?))
+      sleep 0.1
+    end
+    raise self.exception if self.exception
+    raise "Error in job #{self.path}" if self.error? or self.aborted? 
     self
   end
 
-  def produce
-    run
-    join
+  def produce(with_fork: false)
+    if with_fork
+      self.fork
+      self.join
+    else
+      run
+    end
+    self
   end
 
   def load
@@ -233,6 +280,7 @@ class Step
   def step(task_name)
     dependencies.each do |dep|
       return dep if dep.task_name == task_name
+      return dep if dep.overriden_task == task_name
       rec_dep = dep.step(task_name)
       return rec_dep if rec_dep
     end
@@ -250,4 +298,8 @@ class Step
   def fingerprint
     digest_str
   end
+
+  def task_signature
+    [workflow.to_s, task_name] * "#"
+  end
 end

data/lib/scout/workflow/task/dependencies.rb

@@ -0,0 +1,116 @@
+module Task
+  def dependencies(id, provided_inputs, non_default_inputs = [], compute = {})
+    return [] if deps.nil?
+    dependencies = []
+    
+    provided_inputs ||= {}
+
+    # Helper function
+    load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
+      task = step_options.delete(:task) if step_options.include?(:task)
+      workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
+      id = step_options.delete(:id) if step_options.include?(:id)
+      id = step_options.delete(:jobname) if step_options.include?(:jobname)
+
+      step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
+      step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
+
+      resolved_inputs = {}
+      step_inputs.each do |k,v|
+        if Symbol === v
+          input_dep = dependencies.select{|d| d.task_name == v }.first
+          resolved_inputs[k] = input_dep || step_inputs[v] || k
+        else
+          resolved_inputs[k] = v
+        end
+      end
+      job = workflow.job(task, id, resolved_inputs)
+      compute_options = definition_options[:compute] || []
+      compute_options = [compute_options] unless Array === compute_options
+      compute_options << :canfail if definition_options[:canfail]
+      compute_options << :produce if definition_options[:produce]
+      compute_options << :stream if definition_options[:stream]
+      compute[job.path] = compute_options if compute_options.any?
+      [job, step_inputs]
+    end
+
+    # Helper function
+    find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
+      dep_non_default_inputs = dep.non_default_inputs
+      dep_non_default_inputs.select do |name|
+        step_inputs.include?(name)  
+      end
+      dep_non_default_inputs.reject! do |name|
+        definition_options.include?(name) && 
+          (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
+      end
+
+      dep_non_default_inputs
+    end
+
+    deps.each do |workflow,task,definition_options,block=nil|
+      definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
+
+      if provided_inputs.include?(overriden = [workflow.name, task] * "#")
+        dep = provided_inputs[overriden]
+        dep = Step.new dep unless Step === dep
+        dep.type = workflow.tasks[task].type
+        dep.overriden_task = task
+        dep.overriden_workflow = workflow
+        dependencies << dep
+        non_default_inputs << overriden
+        next
+      end
+
+      definition_options ||= {}
+
+      if block
+        fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
+        self.inputs.each do |name,type,desc,value|
+          fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
+        end
+        fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
+        block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
+
+        res = block.call id, block_options, dependencies
+
+        case res
+        when Step
+          dep = res
+          dependencies << dep
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Hash
+          step_options = block_options.merge(res)
+          dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+          dependencies << dep
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Array
+          res.each do |_res|
+            if Hash === _res
+              step_options = block_options.merge(_res)
+              dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+              dependencies << dep
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+              non_default_inputs.concat(dep_non_default_inputs)
+            else
+              dep = _res
+              dependencies << dep
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+              non_default_inputs.concat(dep_non_default_inputs)
+            end
+          end
+        end
+      else
+        step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
+        dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
+        dependencies << dep
+        dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+        non_default_inputs.concat(dep_non_default_inputs)
+      end
+    end
+
+    dependencies
+  end
+end

data/lib/scout/workflow/task/inputs.rb

@@ -3,7 +3,7 @@ module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value
 
-    if String === value && ! [:path, :file, :folder].include?(type) && (options.nil? || (! options[:noload] || options[:stream]))
+    if String === value && ! [:path, :file, :folder, :binary].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
       if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else
@@ -20,7 +20,7 @@ module Task
     end
   end
 
-  def assign_inputs(provided_inputs = {})
+  def assign_inputs(provided_inputs = {}, id = nil)
     if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
       case provided_inputs
       when Array
@@ -43,6 +43,8 @@ module Task
       if ! provided.nil? && provided != value
         non_default_inputs << name.to_sym
         input_array << provided
+      elsif options && options[:jobname]
+        input_array << id
       else
         input_array << value
       end
@@ -53,17 +55,18 @@ module Task
     [input_array, non_default_inputs]
   end
 
-  def process_inputs(provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs provided_inputs
+  def process_inputs(provided_inputs = {}, id = nil)
+    input_array, non_default_inputs = assign_inputs provided_inputs, id
     digest_str = Misc.digest_str(input_array)
     [input_array, non_default_inputs, digest_str]
   end
 
   def save_file_input(orig_file, directory)
+    orig_file = orig_file.path if Step === orig_file
     basename = File.basename(orig_file)
     digest = Misc.digest(orig_file)
     if basename.include? '.'
-      basename.sub!(/(.*)\.(.*)/, "\1-#{digest}.\2")
+      basename.sub!(/(.*)\.(.*)/, '\1-' + digest + '.\2')
     else
       basename += "-#{digest}"
     end
@@ -74,35 +77,51 @@ module Task
   end
 
   def save_inputs(directory, provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs(provided_inputs)
-    self.inputs.each_with_index do |p,i|
+    #input_array, non_default_inputs = assign_inputs(provided_inputs)
+    self.recursive_inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
-      next unless non_default_inputs.include?(name)
+      next unless provided_inputs.include?(name)
+      value = provided_inputs[name]
       input_file = File.join(directory, name.to_s)
 
-      if type == :file
-        relative_file = save_file_input(input_array[i], directory)
-        Persist.save(relative_file, input_file, type)
+      if Path.is_filename?(value) 
+        relative_file = save_file_input(value, directory)
+        Open.write(input_file + ".as_file", relative_file)
+      elsif type == :file
+        relative_file = save_file_input(value, directory)
+        Persist.save(relative_file, input_file, :file)
       elsif type == :file_array
-        new_files = input_array[i].collect do |orig_file|
+        new_files = value.collect do |orig_file|
           save_file_input(orig_file, directory)
         end
         Persist.save(new_files, input_file, type)
+      elsif Open.is_stream?(value)
+        Persist.save(value, input_file, type)
+      elsif Open.has_stream?(value)
+        Persist.save(value.stream, input_file, type)
       else
-        Persist.save(input_array[i], input_file, type)
+        Persist.save(value, input_file, type)
       end
     end
   end
 
   def load_inputs(directory)
-    self.inputs.collect do |p|
+    inputs = IndiferentHash.setup({})
+    self.recursive_inputs.each do |p|
       name, type, desc, value, options = p
       filename = File.join(directory, name.to_s) 
       if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
-        Persist.load(filename, type)
-      else
-        value
+        if filename.end_with?('.as_file')
+          value = Open.read(filename).strip
+          value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
+          inputs[name] = value
+        elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+          inputs[name] = filename
+        else
+          inputs[name] = Persist.load(filename, type)
+        end
       end
     end
+    inputs
   end
 end

data/lib/scout/workflow/task.rb

@@ -2,6 +2,7 @@ require_relative '../meta_extension'
 require_relative '../named_array'
 require_relative 'step'
 require_relative 'task/inputs'
+require_relative 'task/dependencies'
 
 module Task
   extend MetaExtension
@@ -38,123 +39,23 @@ module Task
     binding.instance_exec(*inputs, &self)
   end
 
-  def dependencies(id, provided_inputs, non_default_inputs = [])
-    return [] if deps.nil?
-    dependencies = []
-    
-    provided_inputs ||= {}
-
-    # Helper function
-    load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
-      task = step_options.delete(:task) if step_options.include?(:task)
-      workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
-      id = step_options.delete(:id) if step_options.include?(:id)
-      id = step_options.delete(:jobname) if step_options.include?(:jobname)
-
-      step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
-      step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
-
-      resolved_inputs = {}
-      step_inputs.each do |k,v|
-        if Symbol === v
-          input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || step_inputs[v] || k
-        else
-          resolved_inputs[k] = v
-        end
-      end
-      [workflow.job(task, id, resolved_inputs), step_inputs]
-    end
-
-    # Helper function
-    find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
-      dep_non_default_inputs = dep.non_default_inputs
-      dep_non_default_inputs.select do |name|
-        step_inputs.include?(name)  
-      end
-      dep_non_default_inputs.reject! do |name|
-        definition_options.include?(name) && 
-          (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
-      end
-
-      dep_non_default_inputs
-    end
-
-    deps.each do |workflow,task,definition_options,block=nil|
-      definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
-
-      if provided_inputs.include?(overriden = [workflow.name, task] * "#")
-        dep = provided_inputs[overriden]
-        dep = Step.new dep unless Step === dep
-        dep.type = workflow.tasks[task].type
-        dependencies << dep
-        non_default_inputs << overriden
-        next
-      end
-
-      definition_options ||= {}
-
-      if block
-        fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
-        self.inputs.each do |name,type,desc,value|
-          fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
-        end
-        fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
-        block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
-
-        res = block.call id, block_options, dependencies
-
-        case res
-        when Step
-          dep = res
-          dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
-          non_default_inputs.concat(dep_non_default_inputs)
-        when Hash
-          step_options = block_options.merge(res)
-          dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
-          dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
-          non_default_inputs.concat(dep_non_default_inputs)
-        when Array
-          res.each do |_res|
-            if Hash === _res
-              step_options = block_options.merge(_res)
-              dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
-              dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
-              non_default_inputs.concat(dep_non_default_inputs)
-            else
-              dep = _res
-              dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
-              non_default_inputs.concat(dep_non_default_inputs)
-            end
-          end
-        end
-      else
-        step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
-        dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
-        dependencies << dep
-        dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
-        non_default_inputs.concat(dep_non_default_inputs)
-      end
-    end
-
-    dependencies
-  end
-
   def job(id = DEFAULT_NAME, provided_inputs = nil )
     provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
     provided_inputs = {} if provided_inputs.nil?
+    IndiferentHash.setup(provided_inputs)
     id = DEFAULT_NAME if id.nil?
 
-    inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
+    provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
 
-    dependencies = dependencies(id, provided_inputs, non_default_inputs)
+    inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
+
+    compute = {}
+    dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
     non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
+    non_default_inputs.uniq!
+
     if non_default_inputs.any?
       hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
       name = [id, hash] * "_"
@@ -178,6 +79,7 @@ module Task
       end
     end
 
+
     path = directory[name]
 
     path = path.set_extension(extension) if extension
@@ -189,7 +91,8 @@ module Task
         Log.debug "ID #{self.name} #{id} - Clean"
       end
       NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
-      Step.new path.find, inputs, dependencies, id, non_default_inputs, &self
+      step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
+      Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
     end
   end
 end