scout-gear 10.4.0 → 10.6.1

data/lib/scout/workflow/step.rb

@@ -1,5 +1,6 @@
 require 'scout/path'
 require 'scout/persist'
+require 'scout/semaphore'
 require_relative 'step/info'
 require_relative 'step/status'
 require_relative 'step/load'
@@ -9,11 +10,13 @@ require_relative 'step/provenance'
 require_relative 'step/config'
 require_relative 'step/progress'
 require_relative 'step/inputs'
+require_relative 'step/children'
+require_relative 'step/archive'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
+  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context = nil, &task)
     @path = path
     @inputs = inputs
     @dependencies = dependencies
@@ -24,6 +27,7 @@ class Step
     @task = task
     @mutex = Mutex.new
     @tee_copies = 1
+    @exec_context = exec_context || self
   end
 
   def synchronize(&block)
@@ -44,10 +48,11 @@ class Step
 
   def dependencies
     @dependencies ||= begin
-                        if Open.exists?(info_file)
+                        if info_file && Open.exists?(info_file) && info[:dependencies]
                           info[:dependencies].collect do |path|
+                            path = path.last if Array === path
                             Step.load(path)
-                          end if info[:dependencies]
+                          end
                         else
                           []
                         end
@@ -56,7 +61,7 @@ class Step
 
   attr_accessor :type
   def type
-    @type ||= @task.respond_to?(:type) ? @task.type : info[:type]
+    @type ||= (@task.respond_to?(:type) && @task.type) ? @task.type : info[:type]
   end
 
   def name
@@ -70,7 +75,9 @@ class Step
   def clean_name
     return @id if @id
     return info[:clean_name] if info.include? :clean_name
-    return m[1] if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
+    if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
+      return m[1] 
+    end
     return name.split(".").first
   end
 
@@ -82,10 +89,16 @@ class Step
 
   def workflow
     @workflow ||= @task.workflow if Task === @task
-    @workflow ||= info[:workflow] if Open.exist?(info_file)
+    @workflow ||= info[:workflow] if info_file && Open.exist?(info_file)
     @workflow ||= path.split("/")[-3]
   end
 
+  def full_task_name
+    return nil if task_name.nil?
+    return task_name.to_s if workflow.nil?
+    [workflow, task_name] * "#"
+  end
+
   def exec
 
     if inputs 
@@ -107,7 +120,7 @@ class Step
 
     @result = begin
                 @in_exec = true
-                self.instance_exec(*inputs, &task)
+                @exec_context.instance_exec(*inputs, &task)
               ensure
                 @in_exec = false
               end
@@ -127,23 +140,37 @@ class Step
   def run(stream = false)
     return @result || self.load if done?
     prepare_dependencies
-    run_dependencies
     begin
-      @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
-        clear_info
+
+      case stream
+      when TrueClass, :stream
+        no_load = :stream
+      when :no_load
+        no_load = true
+      else
+        no_load = false
+      end
+
+      @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies, no_load: no_load) do
         input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
-        merge_info :status => :start, :start => Time.now,
+
+
+        reset_info :status => :setup, :issued => Time.now,
           :pid => Process.pid, :pid_hostname => Misc.hostname, 
           :task_name => task_name, :workflow => workflow.to_s,
-          :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
-          :dependencies => dependencies.collect{|d| d.path }
+          :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
+          :dependencies => (dependencies || []) .collect{|d| d.path }
 
+        run_dependencies
+
+        set_info :start, Time.now
+        log :start
         @exec_result = exec
 
         if @exec_result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
           Open.mv self.tmp_path, self.path
         else
-          @exec_result = @exec_result.stream if @exec_result.respond_to?(:stream)
+          @exec_result = @exec_result.stream if @exec_result.respond_to?(:stream) && ! (TSV === @exec_result)
         end
 
         @exec_result
@@ -155,13 +182,25 @@ class Step
           @exec_result
         end
       end
+
+      if TrueClass === no_load
+        consume_all_streams if streaming?
+        @result = nil
+      elsif no_load && ! (IO === @result)
+        @result = nil
+      end
+
+      @result
     rescue Exception => e
-      merge_info :status => :error, :exception => e, :end => Time.now
-      abort_dependencies
-      raise e
+      merge_info :status => :error, :exception => e, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
+      begin
+        abort_dependencies
+      ensure
+        raise e
+      end
     ensure
       if ! (error? || aborted?)
-        if streaming?
+        if @result && streaming?
           ConcurrentStream.setup(@result) do
             merge_info :status => :done, :end => Time.now
           end
@@ -187,12 +226,22 @@ class Step
     end
   end
 
-  def fork
-    Process.fork do
-      clear_info unless present?
-      run(false)
+  def fork(noload = false, semaphore = nil)
+    pid = Process.fork do
+      Signal.trap(:TERM) do
+        raise Aborted, "Recieved TERM Signal on forked process #{Process.pid}"
+      end
+      reset_info status: :queue, pid: Process.pid unless present?
+      if semaphore
+        ScoutSemaphore.synchronize(semaphore) do
+          run(noload)
+        end
+      else
+        run(noload)
+      end
       join
     end
+    Process.detach pid
     grace
     self
   end
@@ -272,17 +321,23 @@ class Step
     while @result.nil? && (present? && ! (terminated? || done?))
       sleep 0.1
     end
+
+    Misc.wait_child info[:pid] if info[:pid]
+
     raise self.exception if self.exception
+
     raise "Error in job #{self.path}" if self.error? or self.aborted? 
+
     self
   end
 
   def produce(with_fork: false)
+    clean if error? && recoverable_error?
     if with_fork
       self.fork
       self.join
     else
-      run
+      run(:no_load)
     end
     self
   end
@@ -305,7 +360,7 @@ class Step
   end
 
   def short_path
-    Scout.identify @path
+    Resource.identify @path
   end
 
   def digest_str

data/lib/scout/workflow/task/dependencies.rb

@@ -19,7 +19,7 @@ module Task
       step_inputs.each do |k,v|
         if Symbol === v
           input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || k
+          resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || v
         else
           resolved_inputs[k] = v
         end
@@ -31,6 +31,9 @@ module Task
       compute_options << :produce if definition_options[:produce]
       compute_options << :stream if definition_options[:stream]
       compute[job.path] = compute_options if compute_options.any?
+
+      job.overriden = false if definition_options[:not_overriden]
+
       [job, step_inputs]
     end
 

data/lib/scout/workflow/task/inputs.rb

@@ -40,7 +40,16 @@ module Task
       input_names << name
       provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
       provided = Task.format_input(provided, type, options || {})
-      if ! provided.nil? && provided != value
+      if provided == value
+        same_as_default = true
+      elsif String === provided && Symbol === value && provided == value.to_s
+        same_as_default = true
+      elsif String === value && Symbol === provided && provided.to_s == value
+        same_as_default = true
+      else
+      same_as_default = false
+      end
+      if ! provided.nil? && ! same_as_default
         non_default_inputs << name.to_sym
         input_array << provided
       elsif options && options[:jobname]
@@ -61,7 +70,7 @@ module Task
     [input_array, non_default_inputs, digest_str]
   end
 
-  def save_file_input(orig_file, directory)
+  def self.save_file_input(orig_file, directory)
     orig_file = orig_file.path if Step === orig_file
     basename = File.basename(orig_file)
     digest = Misc.digest(orig_file)
@@ -76,38 +85,65 @@ module Task
     relative_file
   end
 
+  def self.save_input(directory, name, type, value)
+    input_file = File.join(directory, name.to_s)
+
+    if Path.is_filename?(value) 
+      if type == :path
+        Open.write(input_file + ".as_path", value)
+      else
+        relative_file = save_file_input(value, directory)
+        Open.write(input_file + ".as_file", relative_file)
+      end
+    elsif Step === value
+      Open.write(input_file + ".as_step", value.short_path)
+    elsif type == :file
+      relative_file = save_file_input(value, directory)
+      Persist.save(relative_file, input_file, :file)
+    elsif type == :file_array
+      new_files = value.collect do |orig_file|
+        save_file_input(orig_file, directory)
+      end
+      Persist.save(new_files, input_file, type)
+    elsif Open.is_stream?(value)
+      Open.sensible_write(input_file, value)
+    elsif Open.has_stream?(value)
+      Open.sensible_write(input_file, value.stream)
+    else
+      Persist.save(value, input_file, type)
+    end
+  end
+
   def save_inputs(directory, provided_inputs = {})
-    #input_array, non_default_inputs = assign_inputs(provided_inputs)
     self.recursive_inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
       next unless provided_inputs.include?(name)
       value = provided_inputs[name]
-      input_file = File.join(directory, name.to_s)
-
-      if Path.is_filename?(value) 
-        if type == :path
-          Open.write(input_file + ".as_path", value)
-        else
-          relative_file = save_file_input(value, directory)
-          Open.write(input_file + ".as_file", relative_file)
-        end
-      elsif Step === value
-        Open.write(input_file + ".as_step", value.short_path)
-      elsif type == :file
-        relative_file = save_file_input(value, directory)
-        Persist.save(relative_file, input_file, :file)
-      elsif type == :file_array
-        new_files = value.collect do |orig_file|
-          save_file_input(orig_file, directory)
-        end
-        Persist.save(new_files, input_file, type)
-      elsif Open.is_stream?(value)
-        Persist.save(value, input_file, type)
-      elsif Open.has_stream?(value)
-        Persist.save(value.stream, input_file, type)
+
+      Task.save_input(directory, name, type, value)
+    end
+  end
+
+
+  def self.load_input_from_file(filename, type, options = nil)
+    if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
+      if filename.end_with?('.as_file')
+        value = Open.read(filename).strip
+        value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
+        value
+      elsif filename.end_with?('.as_step')
+        value = Open.read(filename).strip
+        Step.load value
+      elsif filename.end_with?('.as_path')
+        value = Open.read(filename).strip
+        Path.setup value
+      elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+        filename
       else
-        Persist.save(value, input_file, type)
+        Persist.load(filename, type)
       end
+    else
+      return nil
     end
   end
 
@@ -116,21 +152,35 @@ module Task
     self.recursive_inputs.each do |p|
       name, type, desc, value, options = p
       filename = File.join(directory, name.to_s) 
-      if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
-        if filename.end_with?('.as_file')
-          value = Open.read(filename).strip
-          value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
-          inputs[name] = value
-        elsif filename.end_with?('.as_step')
-          value = Open.read(filename).strip
-          inputs[name] = Step.load value
-        elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
-          inputs[name] = filename
-        else
-          inputs[name] = Persist.load(filename, type)
-        end
-      end
+      value = Task.load_input_from_file(filename, type, options)
+      inputs[name] = value unless value.nil?
+    end
+
+    Dir.glob(File.join(directory, "*#*")).each do |file|
+      override_dep, _, extension = File.basename(file).partition(".")
+
+      inputs[override_dep] = Task.load_input_from_file(file, :file)
     end
+
     inputs
   end
+
+  def recursive_inputs(overriden = [])
+    return inputs.dup if deps.nil?
+    deps.inject(inputs.dup) do |acc,dep|
+      workflow, task, options = dep
+      next acc if workflow.nil? || task.nil?
+      next acc if overriden.include?([workflow.name, task.to_s] * "#")
+      overriden.concat options.keys.select{|k| k.to_s.include?("#") } if options
+
+      workflow.tasks[task].recursive_inputs(overriden).dup.each do |info|
+        name, _ = info
+        next if options.include?(name.to_sym) || options.include?(name.to_s)
+        acc << info
+      end
+
+      acc
+    end
+  end
+
 end

data/lib/scout/workflow/task.rb

@@ -1,12 +1,12 @@
-require 'scout/meta_extension'
+require 'scout/annotation'
 require 'scout/named_array'
 require_relative 'step'
 require_relative 'task/inputs'
 require_relative 'task/dependencies'
 
 module Task
-  extend MetaExtension
-  extension_attr :name, :type, :inputs, :deps, :directory, :description, :returns, :extension, :workflow
+  extend Annotation
+  annotation :name, :type, :inputs, :deps, :directory, :description, :returns, :extension, :workflow
 
   DEFAULT_NAME = "Default"
 
@@ -22,15 +22,6 @@ module Task
     @inputs ||= []
   end
 
-  def recursive_inputs
-    return inputs if deps.nil?
-    deps.inject(inputs) do |acc,dep|
-      workflow, task = dep
-      next acc if workflow.nil? || task.nil?
-      acc += workflow.tasks[task].recursive_inputs
-    end
-  end
-
   def directory
     @directory ||= Task.default_directory
   end
@@ -39,75 +30,81 @@ module Task
     binding.instance_exec(*inputs, &self)
   end
 
-  def job(id = DEFAULT_NAME, provided_inputs = nil )
-    provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
-    provided_inputs = {} if provided_inputs.nil?
-    IndiferentHash.setup(provided_inputs)
-    id = DEFAULT_NAME if id.nil?
+  def job(id = nil, provided_inputs = nil)
+    Persist.memory("Task job", other_options: {task: self, id: id, provided_inputs: provided_inputs}) do 
+      provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
+      provided_inputs = {} if provided_inputs.nil?
+      IndiferentHash.setup(provided_inputs)
 
+      if id.nil?
+        inputs.each do |name,type,desc,default,input_options|
+          next unless input_options && input_options[:jobname]
+          id = provided_inputs[name] || default
+        end
+        id = DEFAULT_NAME if id.nil?
+      end
 
-    missing_inputs = []
-    self.inputs.each do |input,type,desc,val,options|
-      next unless options && options[:required]
-      missing_inputs << input unless provided_inputs.include?(input)
-    end if self.inputs
+      missing_inputs = []
+      self.inputs.each do |input,type,desc,val,options|
+        next unless options && options[:required]
+        missing_inputs << input unless provided_inputs.include?(input)
+      end if self.inputs
 
-    if missing_inputs.length == 1
-      raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
-    end
+      if missing_inputs.length == 1
+        raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
+      end
 
-    if missing_inputs.length > 1
-      raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
-    end
+      if missing_inputs.length > 1
+        raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
+      end
 
-    provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
+      provided_inputs = load_inputs(provided_inputs.delete(:load_inputs)).merge(provided_inputs) if Hash === provided_inputs && provided_inputs[:load_inputs]
 
-    inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
+      job_inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
 
-    compute = {}
-    dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
+      compute = {}
+      dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
-    #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
+      #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
-    non_default_inputs.uniq!
+      non_default_inputs.uniq!
 
-    if non_default_inputs.any?
-      hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
-      name = [id, hash] * "_"
-    else
-      name = id
-    end
+      if non_default_inputs.any?
+        hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
+        name = [id, hash] * "_"
+      else
+        name = id
+      end
 
-    extension = self.extension
-    if extension == :dep_task
-      extension = nil
-      if dependencies.any?
-        dep_basename = File.basename(dependencies.last.path)
-        if dep_basename.include? "."
-          parts = dep_basename.split(".")
-          extension = [parts.pop]
-          while parts.last.length <= 4
-            extension << parts.pop
+      extension = self.extension
+      if extension == :dep_task
+        extension = nil
+        if dependencies.any?
+          dep_basename = File.basename(dependencies.last.path)
+          if dep_basename.include? "."
+            parts = dep_basename.split(".")
+            extension = [parts.pop]
+            while parts.last.length <= 4
+              extension << parts.pop
+            end
+            extension = extension.reverse * "."
           end
-          extension = extension.reverse * "."
         end
       end
-    end
 
 
-    path = directory[name]
+      path = directory[name]
 
-    path = path.set_extension(extension) if extension
+      path = path.set_extension(extension) if extension
 
-    Persist.memory(path) do 
       if hash
         Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
       else
         Log.debug "ID #{self.name} #{id} - Clean"
       end
-      NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
+      NamedArray.setup(job_inputs, @inputs.collect{|i| i[0] }) if @inputs
       step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
-      Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
+      Step.new path.find, job_inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
     end
   end
 

data/lib/scout/workflow/usage.rb

@@ -114,6 +114,7 @@ module Task
   def get_SOPT
     sopt_option_string = self.SOPT_str
     job_options = SOPT.get sopt_option_string
+
     recursive_inputs.uniq.each do |name,type|
       next unless type.to_s.include?('array')
       if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
@@ -285,7 +286,6 @@ module Workflow
         task = self.tasks[task_name]
       end
 
-      #dependencies = self.rec_dependencies(task_name).collect{|dep_name| Array === dep_name ? dep_name.first.tasks[dep_name[1].to_sym] : self.tasks[dep_name.to_sym]}
       str.puts task.usage(self, self.recursive_deps(task_name))
 
       dep_tree = {[self, task_name] => dep_tree(task_name)}

data/lib/scout/workflow/util.rb

@@ -8,6 +8,10 @@ module Workflow
     mod
   end
 
+  def self.installed_workflows
+    Path.setup("workflows").glob_all("*").collect{|f| File.basename(f) }.uniq
+  end
+
   def find_in_dependencies(name, dependencies)
     name = name.to_sym
     dependencies.select{|dep| dep.task_name.to_sym == name }