scout-gear 7.3.0 → 8.1.0

data/lib/scout/workflow/step.rb

@@ -1,6 +1,7 @@
 require_relative '../path'
 require_relative '../persist'
 require_relative 'step/info'
+require_relative 'step/status'
 require_relative 'step/load'
 require_relative 'step/file'
 require_relative 'step/dependencies'
@@ -10,11 +11,14 @@ require_relative 'step/progress'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
-  def initialize(path, inputs = nil, dependencies = nil, &task) 
+  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :compute
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, compute = nil, &task) 
     @path = path
     @inputs = inputs
     @dependencies = dependencies
+    @id = id
+    @non_default_inputs = non_default_inputs
+    @compute = compute 
     @task = task
     @mutex = Mutex.new
     @tee_copies = 1
@@ -26,7 +30,7 @@ class Step
 
   def inputs
     @inputs ||= begin
-                  if Open.exists?(info_file)
+                  if info_file && Open.exists?(info_file)
                     info[:inputs]
                   else
                     []
@@ -39,7 +43,7 @@ class Step
                         if Open.exists?(info_file)
                           info[:dependencies].collect do |path|
                             Step.load(path)
-                          end
+                          end if info[:dependencies]
                         else
                           []
                         end
@@ -55,6 +59,13 @@ class Step
     @name ||= File.basename(@path)
   end
 
+  def clean_name
+    return @id if @id
+    return info[:clean_name] if info.include? :clean_name
+    return m[1] if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
+    return name.split(".").first
+  end
+
   def task_name
     @task_name ||= @task.name if @task.respond_to?(:name)
   end
@@ -64,26 +75,76 @@ class Step
   end
 
   def exec
-    @result = self.instance_exec(*inputs, &task)
+
+    if inputs 
+      if Task === task
+        types = task.inputs.collect{|name,type| type }
+        new_inputs = inputs.zip(types).collect{|input,info|  
+          type, desc, default, options = info
+          next input unless Step === input
+          input.join if input.streaming?
+          Task.format_input(input.join.path, type, options)
+        }
+      else
+        new_inputs = inputs.collect{|input|  
+          Step === input ? input.load : input
+        }
+      end
+      inputs = new_inputs
+    end
+
+    @result = begin
+                @in_exec = true
+                self.instance_exec(*inputs, &task)
+              ensure
+                @in_exec = false
+              end
+  end
+
+  def tmp_path
+    @tmp_path ||= begin
+                    basename = File.basename(@path)
+                    dirname = File.dirname(@path)
+                    tmp_path = File.join(dirname, '.' + basename)
+                    @path.setup(tmp_path) if Path === @path
+                    tmp_path
+                  end
   end
 
   attr_reader :result
-  def run
+  def run(stream = false)
     return @result || self.load if done?
     prepare_dependencies
     run_dependencies
-    @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
+    @result = 
       begin
-        merge_info :status => :start, :start => Time.now,
-          :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
-          :inputs => inputs, :type => type,
-          :dependencies => dependencies.collect{|d| d.path }
-
-        @result = exec
-        @result = @result.respond_to?(:stream) ? @result.stream : @result
-        @result
+        Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
+          clear_info
+          merge_info :status => :start, :start => Time.now,
+            :pid => Process.pid, :pid_hostname => Misc.hostname, 
+            :inputs => MetaExtension.purge(inputs), :type => type,
+            :dependencies => dependencies.collect{|d| d.path }
+
+          @result = exec
+
+          if @result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
+            Open.mv self.tmp_path, self.path
+          else
+            @result = @result.stream if @result.respond_to?(:stream)
+          end
+
+          @result
+
+          if (IO === @result || StringIO === @result) && (ENV["SCOUT_NO_STREAM"] == "true" || ! stream)
+            Open.sensible_write(self.path, @result)
+            @result = nil
+          else
+            @result
+          end
+        end
       rescue Exception => e
-        merge_info :status => :error, :exception => e
+        merge_info :status => :error, :exception => e, :end => Time.now
+        abort_dependencies
         raise e
       ensure
         if ! (error? || aborted?)
@@ -93,10 +154,14 @@ class Step
             end
 
             @result.abort_callback = proc do |exception|
-              if Aborted === exception || Interrupt === exception
+              if exception.nil? || Aborted === exception || Interrupt === exception
                 merge_info :status => :aborted, :end => Time.now
               else
-                merge_info :status => :error, :exception => exception, :end => Time.now
+                begin
+                  merge_info :status => :error, :exception => exception, :end => Time.now
+                rescue Exception
+                  Log.exception $!
+                end
               end
             end
 
@@ -106,8 +171,16 @@ class Step
           end
         end
       end
+  end
+
+  def fork
+    Process.fork do
+      clear_info unless present?
+      run(false)
+      join
     end
-    @result
+    grace
+    self
   end
 
   def done?
@@ -126,29 +199,74 @@ class Step
         else
           Log.debug "Taking result #{Log.fingerprint @result}"
         end
+
         @take_stream, @result = @result, @result.next
-        @take_stream
-      elsif done?
+
+        return @take_stream
+      end
+    end
+
+    if done?
+      Open.open(self.path)
+    else
+      if running?
+        join
         Open.open(self.path)
       else
-        if running?
-          nil
-        else
-          exec
-        end
+        exec
+      end
+    end
+  end
+
+  def consume_all_streams
+    threads = [] 
+    while @result && streaming? && stream = self.stream
+      threads << Open.consume_stream(stream, true)
+    end
+    threads.each do |t| 
+      begin
+        t.join 
+      rescue Exception
+        threads.each{|t| t.raise(Aborted); t.join }
+        raise $!
       end
     end
   end
 
+  def present?
+    Open.exist?(path) ||
+      Open.exist?(info_file) ||
+      Open.exist?(files_dir)
+  end
+
+  def grace
+    while ! present?
+      sleep 0.1
+    end
+  end
+
+  def terminated?
+    ! @in_exec && (done? || error? || aborted?)
+  end
+
   def join
-    io = self.stream if streaming?
-    Open.consume_stream(io, false) if io
+    consume_all_streams
+    while @result.nil? && (present? && ! terminated?)
+      sleep 0.1
+    end
+    raise self.exception if self.exception
+    raise "Error in job #{self.path}" if self.error? or self.aborted? 
     self
   end
 
-  def produce
-    run
-    join
+  def produce(with_fork: false)
+    if with_fork
+      self.fork
+      self.join
+    else
+      run
+    end
+    self
   end
 
   def load
@@ -157,31 +275,24 @@ class Step
     done? ? Persist.load(path, type) : exec
   end
 
-  def clean
-    @take_stream = nil 
-    @result = nil
-    @info = nil
-    @info_load_time = nil
-    Open.rm path if Open.exist?(path)
-    Open.rm info_file if Open.exist?(info_file)
-    Open.rm_rf files_dir if Open.exist?(files_dir)
-  end
-
-  def recursive_clean
-    dependencies.each do |dep|
-      dep.recursive_clean
-    end
-    clean
-  end
-
   def step(task_name)
     dependencies.each do |dep|
       return dep if dep.task_name == task_name
+      rec_dep = dep.step(task_name)
+      return rec_dep if rec_dep
     end
     nil
   end
 
+  def short_path
+    Scout.identify @path
+  end
+
   def digest_str
-    path.dup
+    "Step: " + short_path
+  end
+
+  def fingerprint
+    digest_str
   end
 end

data/lib/scout/workflow/task/dependencies.rb

@@ -0,0 +1,114 @@
+module Task
+  def dependencies(id, provided_inputs, non_default_inputs = [], compute = {})
+    return [] if deps.nil?
+    dependencies = []
+    
+    provided_inputs ||= {}
+
+    # Helper function
+    load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
+      task = step_options.delete(:task) if step_options.include?(:task)
+      workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
+      id = step_options.delete(:id) if step_options.include?(:id)
+      id = step_options.delete(:jobname) if step_options.include?(:jobname)
+
+      step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
+      step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
+
+      resolved_inputs = {}
+      step_inputs.each do |k,v|
+        if Symbol === v
+          input_dep = dependencies.select{|d| d.task_name == v }.first
+          resolved_inputs[k] = input_dep || step_inputs[v] || k
+        else
+          resolved_inputs[k] = v
+        end
+      end
+      job = workflow.job(task, id, resolved_inputs)
+      compute_options = definition_options[:compute] || []
+      compute_options = [compute_options] unless Array === compute_options
+      compute_options << :canfail if definition_options[:canfail]
+      compute_options << :produce if definition_options[:produce]
+      compute_options << :stream if definition_options[:stream]
+      compute[job.path] = compute_options if compute_options.any?
+      [job, step_inputs]
+    end
+
+    # Helper function
+    find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
+      dep_non_default_inputs = dep.non_default_inputs
+      dep_non_default_inputs.select do |name|
+        step_inputs.include?(name)  
+      end
+      dep_non_default_inputs.reject! do |name|
+        definition_options.include?(name) && 
+          (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
+      end
+
+      dep_non_default_inputs
+    end
+
+    deps.each do |workflow,task,definition_options,block=nil|
+      definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
+
+      if provided_inputs.include?(overriden = [workflow.name, task] * "#")
+        dep = provided_inputs[overriden]
+        dep = Step.new dep unless Step === dep
+        dep.type = workflow.tasks[task].type
+        dependencies << dep
+        non_default_inputs << overriden
+        next
+      end
+
+      definition_options ||= {}
+
+      if block
+        fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
+        self.inputs.each do |name,type,desc,value|
+          fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
+        end
+        fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
+        block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
+
+        res = block.call id, block_options, dependencies
+
+        case res
+        when Step
+          dep = res
+          dependencies << dep
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Hash
+          step_options = block_options.merge(res)
+          dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+          dependencies << dep
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Array
+          res.each do |_res|
+            if Hash === _res
+              step_options = block_options.merge(_res)
+              dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+              dependencies << dep
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+              non_default_inputs.concat(dep_non_default_inputs)
+            else
+              dep = _res
+              dependencies << dep
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+              non_default_inputs.concat(dep_non_default_inputs)
+            end
+          end
+        end
+      else
+        step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
+        dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
+        dependencies << dep
+        dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+        non_default_inputs.concat(dep_non_default_inputs)
+      end
+    end
+
+    dependencies
+  end
+end

data/lib/scout/workflow/task/inputs.rb

@@ -1,11 +1,10 @@
+require_relative '../../named_array'
 module Task
+  def self.format_input(value, type, options = {})
+    return value if IO === value || StringIO === value || Step === value
 
-  def format_input(value, type, options = {})
-    return value if IO === value || StringIO === value
-
-    value = value.load if Step === value
-    if String === value && ! [:path, :file].include?(type) 
-      if Open.exists?(value)
+    if String === value && ! [:path, :file, :folder, :binary].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+      if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else
         Persist.deserialize(value, type)
@@ -13,7 +12,7 @@ module Task
     else
       if m = type.to_s.match(/(.*)_array/)
         if Array === value
-          value.collect{|v| format_input(v, m[1].to_sym, options) }
+          value.collect{|v| self.format_input(v, m[1].to_sym, options) }
         end
       else
         value
@@ -34,42 +33,37 @@ module Task
     IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
 
     input_array = []
+    input_names = []
     non_default_inputs = []
     self.inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
+      input_names << name
       provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
-      provided = format_input(provided, type, options || {})
+      provided = Task.format_input(provided, type, options || {})
       if ! provided.nil? && provided != value
-        non_default_inputs << name
+        non_default_inputs << name.to_sym
         input_array << provided
       else
         input_array << value
       end
     end
 
+    NamedArray.setup(input_array, input_names)
+
     [input_array, non_default_inputs]
   end
 
-  def digest_inputs(provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs(provided_inputs)
-    if Array === provided_inputs 
-      Misc.digest(input_array)
-    else
-      Misc.digest(input_array)
-    end
-  end
-  
   def process_inputs(provided_inputs = {})
     input_array, non_default_inputs = assign_inputs provided_inputs
-    digest = Misc.digest(input_array)
-    [input_array, non_default_inputs, digest]
+    digest_str = Misc.digest_str(input_array)
+    [input_array, non_default_inputs, digest_str]
   end
 
   def save_file_input(orig_file, directory)
     basename = File.basename(orig_file)
     digest = Misc.digest(orig_file)
     if basename.include? '.'
-      basename.sub!(/(.*)\.(.*)/, "\1-#{digest}.\2")
+      basename.sub!(/(.*)\.(.*)/, '\1-' + digest + '.\2')
     else
       basename += "-#{digest}"
     end
@@ -80,35 +74,49 @@ module Task
   end
 
   def save_inputs(directory, provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs(provided_inputs)
-    self.inputs.each_with_index do |p,i|
+    #input_array, non_default_inputs = assign_inputs(provided_inputs)
+    self.recursive_inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
-      next unless non_default_inputs.include?(name)
+      next unless provided_inputs.include?(name)
+      value = provided_inputs[name]
       input_file = File.join(directory, name.to_s)
 
       if type == :file
-        relative_file = save_file_input(input_array[i], directory)
-        Persist.save(relative_file, input_file, type)
+        relative_file = save_file_input(value, directory)
+        Persist.save(relative_file, input_file, :file)
       elsif type == :file_array
-        new_files = input_array[i].collect do |orig_file|
+        new_files = value.collect do |orig_file|
           save_file_input(orig_file, directory)
         end
         Persist.save(new_files, input_file, type)
+      elsif Path.is_filename?(value) 
+        relative_file = save_file_input(value, directory)
+        Open.write(input_file + ".as_file", relative_file)
+      elsif Open.is_stream?(value)
+        Persist.save(input_file, value, type)
+      elsif Open.has_stream?(value)
+        Persist.save(input_file, value.stream, type)
       else
-        Persist.save(input_array[i], input_file, type)
+        Persist.save(value, input_file, type)
       end
     end
   end
 
   def load_inputs(directory)
-    self.inputs.collect do |p|
+    inputs = IndiferentHash.setup({})
+    self.recursive_inputs.each do |p|
       name, type, desc, value, options = p
       filename = File.join(directory, name.to_s) 
       if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
-        Persist.load(filename, type)
-      else
-        value
+        if filename.end_with?('.as_file')
+          value = Open.read(filename).strip
+          value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
+          inputs[name] = value
+        else
+          inputs[name] = Persist.load(filename, type)
+        end
       end
     end
+    inputs
   end
 end