scout-gear 8.1.0 → 9.1.0

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -6,7 +6,7 @@ module Workflow
       return workload if job.done? && job.updated?
 
       job.dependencies.each do |dep|
-        next if dep.done? && job.updated?
+        next if dep.done? && dep.updated?
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep
@@ -14,7 +14,7 @@ module Workflow
       end
 
       job.input_dependencies.each do |dep|
-        next if dep.done? && job.updated?
+        next if dep.done? && dep.updated?
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep
@@ -85,15 +85,15 @@ module Workflow
         candidates = workload.
           select{|k,v| v.empty? }.
           collect{|k,v| k }.
-          reject{|k| k.done? }
+          reject{|k| k.done? || k.running? }
       else
         candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
           select{|k,v| v.empty? }.
           collect{|k,v| k }.
-          reject{|k| k.done? }
+          reject{|k| k.done? || k.running? }
       end
 
-      top_level = workload.keys - workload.values.flatten
+      #top_level = workload.keys - workload.values.flatten
 
       candidates = purge_duplicates candidates
       candidates = sort_candidates candidates, rules
@@ -192,7 +192,10 @@ module Workflow
         workload = Orchestrator.workload(jobs)
         all_jobs = workload.keys
 
+        all_jobs.each{|job| job.clean unless job.done? && job.updated? }
+
         top_level_jobs = jobs.collect{|job| job.path }
+        failed_jobs = []
         while workload.any? 
 
           candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
@@ -203,9 +206,15 @@ module Workflow
             case 
             when (job.error? || job.aborted?)
               begin
-                if job.recoverable_error?
-                  job.clean
-                  raise TryAgain
+                if job.recoverable_error? 
+                  if failed_jobs.include?(job)
+                    Log.warn "Failed twice #{job.path} with recoverable error"
+                    next
+                  else
+                    failed_jobs << job
+                    job.clean
+                    raise TryAgain
+                  end
                 else
                   next
                 end

data/lib/scout/workflow/step/dependencies.rb

@@ -1,11 +1,18 @@
 class Step
-  def rec_dependencies
-    rec_dependencies = dependencies.dup
-    dependencies.inject(rec_dependencies){|acc,d| acc.concat d.rec_dependencies }
+  def rec_dependencies(connected = false, seen = [])
+    direct_deps = []
+    dependencies.each do |dep|
+      next if seen.include? dep.path
+      next if connected && dep.done? && dep.updated?
+      direct_deps << dep
+    end
+    seen.concat direct_deps.collect{|d| d.path }
+    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
   end
 
   def recursive_inputs
-    recursive_inputs = @inputs.to_hash
+    recursive_inputs = NamedArray === @inputs ? @inputs.to_hash : {}
+    return recursive_inputs if dependencies.nil?
     dependencies.inject(recursive_inputs) do |acc,dep|
       acc.merge(dep.recursive_inputs)
     end

data/lib/scout/workflow/step/file.rb

@@ -2,7 +2,11 @@ class Step
   def files_dir
     @files_dir ||= begin
                      dir = @path + ".files"
-                     @path.annotate(dir) if Path === @path
+                     if Path === @path
+                       @path.annotate(dir)
+                     else
+                       Path.setup(dir)
+                     end
                      dir.pkgdir = self
                      dir
                    end

data/lib/scout/workflow/step/info.rb

@@ -9,12 +9,18 @@ class Step
                    end
   end
 
+  def self.load_info(info_file)
+    info = Persist.load(info_file, SERIALIZER) || {}
+    IndiferentHash.setup(info)
+  end
+
   def load_info
-    @info = Persist.load(info_file, SERIALIZER) || {}
-    IndiferentHash.setup(@info)
+    @info = Step.load_info(info_file)
     @info_load_time = Time.now
   end
 
+
+
   def save_info(info = nil)
     Persist.save(info, info_file, SERIALIZER)
     @info_load_time = Time.now
@@ -127,6 +133,14 @@ class Step
     ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
   end
 
+  def overriden?
+    overriden_task || overriden_workflow || dependencies.select{|d| d.overriden? }.any?
+  end
+
+  def overriden_deps
+    rec_dependencies.select{|d| d.overriden? }
+  end
+
   def exception
     info[:exception]
   end

data/lib/scout/workflow/step/inputs.rb

@@ -0,0 +1,5 @@
+class Step
+  def save_inputs(inputs_dir)
+    self.task.save_inputs(inputs_dir, provided_inputs)
+  end
+end

data/lib/scout/workflow/step/provenance.rb

@@ -83,10 +83,25 @@ class Step
       end
     end
 
-    str << "\n"
+    str
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
+  def self.prov_indent(step, offset, input_dependencies)
+    return (" " * (offset))
+    if step.alias?
+      (" " * offset + "🡵")
+    elsif step.overriden_task
+      (" " * offset + "🡇")
+    elsif input_dependencies.include?(step)
+      (" " * offset + "┝")
+    elsif step.input_dependencies.any?
+      (" " * offset + "╰")
+    else
+      (" " * (offset+1))
+    end
+  end
+
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil, input_dependencies = nil)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -101,7 +116,7 @@ class Step
 
     this_step_msg = prov_report_msg(status, name, path, info, input)
 
-    input_dependencies = {}
+    input_dependencies ||= {}
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
@@ -116,8 +131,10 @@ class Step
       end
     end if step.dependencies
 
-    str = ""
-    str = " " * offset + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
+    str = []
+
+    indent = prov_indent(step, offset, input_dependencies)
+    str << indent + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
     step.dependencies.dup.tap{|l| 
       l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
@@ -126,10 +143,10 @@ class Step
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
+        str.concat(prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep], input_dependencies.dup).split("\n"))
       else
         if expand_repeats
-          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
+          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task, input_dependencies[dep], input_dependencies.dup)))
         else
           info = dep.info  || {}
           status = info[:status] || :missing
@@ -138,13 +155,14 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
+          dep_indent = prov_indent(dep, offset+1, input_dependencies)
+          str << dep_indent + Log.color(Step.status_color(status), Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
         end
       end
     end if step.dependencies
 
-    str += (" " * offset) + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
+    str << indent + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
-    str
+    str * "\n"
   end
 end

data/lib/scout/workflow/step/status.rb

@@ -1,6 +1,6 @@
 class Step
   def abort(exception = nil)
-    if info[:pid] != Process.pid && Misc.alive?(pid)
+    if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
       Process.kill pid
     else
       while @result && streaming? && stream = self.stream
@@ -15,7 +15,7 @@ class Step
 
   def updated?
     return false if self.error? && self.recoverable_error?
-    return true unless ENV["SCOUT_UPDATE"]
+    return true if self.done? && ! ENV["SCOUT_UPDATE"]
     newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
 
@@ -28,10 +28,15 @@ class Step
     @info = nil
     @info_load_time = nil
     Open.rm path if Open.exist?(path)
+    Open.rm tmp_path if Open.exist?(tmp_path)
     Open.rm info_file if Open.exist?(info_file)
     Open.rm_rf files_dir if Open.exist?(files_dir)
   end
 
+  def self.clean(file)
+    Step.new(file).clean
+  end
+
 
   def recursive_clean
     dependencies.each do |dep|
@@ -40,4 +45,23 @@ class Step
     clean
   end
 
+  def canfail?
+    @compute && @compute.include?(:canfail)
+  end
+
+  def started?
+    return true if done?
+    return false unless Open.exist?(info_file)
+    pid = info[:pid]
+    return false unless pid
+    return Misc.pid_alive?(pid)
+  end
+
+  def waiting?
+    present? and not started?
+  end
+
+  def dirty?
+    done? && ! updated?
+  end
 end

data/lib/scout/workflow/step.rb

@@ -8,16 +8,18 @@ require_relative 'step/dependencies'
 require_relative 'step/provenance'
 require_relative 'step/config'
 require_relative 'step/progress'
+require_relative 'step/inputs'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :compute
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, compute = nil, &task) 
+  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
     @path = path
     @inputs = inputs
     @dependencies = dependencies
     @id = id
     @non_default_inputs = non_default_inputs
+    @provided_inputs = provided_inputs
     @compute = compute 
     @task = task
     @mutex = Mutex.new
@@ -31,7 +33,9 @@ class Step
   def inputs
     @inputs ||= begin
                   if info_file && Open.exists?(info_file)
-                    info[:inputs]
+                    inputs = info[:inputs]
+                    NamedArray.setup(inputs, info[:input_names]) if inputs && info[:input_names]
+                    inputs
                   else
                     []
                   end
@@ -59,6 +63,10 @@ class Step
     @name ||= File.basename(@path)
   end
 
+  def short_path
+    [workflow.to_s, task_name, name] * "/"
+  end
+
   def clean_name
     return @id if @id
     return info[:clean_name] if info.include? :clean_name
@@ -120,9 +128,10 @@ class Step
       begin
         Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
           clear_info
+          input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
           merge_info :status => :start, :start => Time.now,
             :pid => Process.pid, :pid_hostname => Misc.hostname, 
-            :inputs => MetaExtension.purge(inputs), :type => type,
+            :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
             :dependencies => dependencies.collect{|d| d.path }
 
           @result = exec
@@ -209,7 +218,7 @@ class Step
     if done?
       Open.open(self.path)
     else
-      if running?
+      if running? || waiting?
         join
         Open.open(self.path)
       else
@@ -251,7 +260,7 @@ class Step
 
   def join
     consume_all_streams
-    while @result.nil? && (present? && ! terminated?)
+    while @result.nil? && (present? && ! (terminated? || done?))
       sleep 0.1
     end
     raise self.exception if self.exception
@@ -276,8 +285,10 @@ class Step
   end
 
   def step(task_name)
+    task_name = task_name.to_sym
     dependencies.each do |dep|
-      return dep if dep.task_name == task_name
+      return dep if dep.task_name && dep.task_name.to_sym == task_name
+      return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
       rec_dep = dep.step(task_name)
       return rec_dep if rec_dep
     end
@@ -295,4 +306,12 @@ class Step
   def fingerprint
     digest_str
   end
+
+  def task_signature
+    [workflow.to_s, task_name] * "#"
+  end
+
+  def alias?
+    task.alias? if task
+  end
 end

data/lib/scout/workflow/task/dependencies.rb

@@ -54,7 +54,10 @@ module Task
       if provided_inputs.include?(overriden = [workflow.name, task] * "#")
         dep = provided_inputs[overriden]
         dep = Step.new dep unless Step === dep
+        dep = dep.dup
         dep.type = workflow.tasks[task].type
+        dep.overriden_task = task
+        dep.overriden_workflow = workflow
         dependencies << dep
         non_default_inputs << overriden
         next

data/lib/scout/workflow/task/inputs.rb

@@ -20,7 +20,7 @@ module Task
     end
   end
 
-  def assign_inputs(provided_inputs = {})
+  def assign_inputs(provided_inputs = {}, id = nil)
     if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
       case provided_inputs
       when Array
@@ -43,6 +43,8 @@ module Task
       if ! provided.nil? && provided != value
         non_default_inputs << name.to_sym
         input_array << provided
+      elsif options && options[:jobname]
+        input_array << id
       else
         input_array << value
       end
@@ -53,13 +55,14 @@ module Task
     [input_array, non_default_inputs]
   end
 
-  def process_inputs(provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs provided_inputs
+  def process_inputs(provided_inputs = {}, id = nil)
+    input_array, non_default_inputs = assign_inputs provided_inputs, id
     digest_str = Misc.digest_str(input_array)
     [input_array, non_default_inputs, digest_str]
   end
 
   def save_file_input(orig_file, directory)
+    orig_file = orig_file.path if Step === orig_file
     basename = File.basename(orig_file)
     digest = Misc.digest(orig_file)
     if basename.include? '.'
@@ -81,7 +84,12 @@ module Task
       value = provided_inputs[name]
       input_file = File.join(directory, name.to_s)
 
-      if type == :file
+      if Path.is_filename?(value) 
+        relative_file = save_file_input(value, directory)
+        Open.write(input_file + ".as_file", relative_file)
+      elsif Step === value
+        Open.write(input_file + ".as_step", value.short_path)
+      elsif type == :file
         relative_file = save_file_input(value, directory)
         Persist.save(relative_file, input_file, :file)
       elsif type == :file_array
@@ -89,13 +97,10 @@ module Task
           save_file_input(orig_file, directory)
         end
         Persist.save(new_files, input_file, type)
-      elsif Path.is_filename?(value) 
-        relative_file = save_file_input(value, directory)
-        Open.write(input_file + ".as_file", relative_file)
       elsif Open.is_stream?(value)
-        Persist.save(input_file, value, type)
+        Persist.save(value, input_file, type)
       elsif Open.has_stream?(value)
-        Persist.save(input_file, value.stream, type)
+        Persist.save(value.stream, input_file, type)
       else
         Persist.save(value, input_file, type)
       end
@@ -112,6 +117,11 @@ module Task
           value = Open.read(filename).strip
           value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
           inputs[name] = value
+        elsif filename.end_with?('.as_step')
+          value = Open.read(filename).strip
+          inputs[name] = Step.load value
+        elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+          inputs[name] = filename
         else
           inputs[name] = Persist.load(filename, type)
         end

data/lib/scout/workflow/task.rb

@@ -42,16 +42,32 @@ module Task
   def job(id = DEFAULT_NAME, provided_inputs = nil )
     provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
     provided_inputs = {} if provided_inputs.nil?
+    IndiferentHash.setup(provided_inputs)
     id = DEFAULT_NAME if id.nil?
 
+
+    missing_inputs = []
+    self.inputs.each do |input,type,desc,val,options|
+      next unless options && options[:required]
+      missing_inputs << input unless provided_inputs.include?(input)
+    end if self.inputs
+
+    if missing_inputs.length == 1
+      raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
+    end
+
+    if missing_inputs.length > 1
+      raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
+    end
+
     provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
 
-    inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
+    inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
 
     compute = {}
     dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
-    non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
+    #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
     non_default_inputs.uniq!
 
@@ -90,7 +106,12 @@ module Task
         Log.debug "ID #{self.name} #{id} - Clean"
       end
       NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
-      Step.new path.find, inputs, dependencies, id, non_default_inputs, compute, &self
+      step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
+      Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
     end
   end
+
+  def alias?
+    @extension == :dep_task
+  end
 end

data/lib/scout/workflow/usage.rb

@@ -13,28 +13,30 @@ module Task
         if paragraph
           str.puts "\n" << Misc.format_paragraph(paragraph) 
         end
-        str.puts
       else
         title = self.name.to_s
         str.puts Log.color :yellow, title
         str.puts Log.color :yellow, "-" * title.length
         str.puts "\n" << Misc.format_paragraph(description) 
-        str.puts
       end
     else
       title = self.name.to_s
       str.puts Log.color :yellow, title
       str.puts Log.color :yellow, "-" * title.length
-      str.puts
     end
 
 
     selects = []
+
     if inputs && inputs.any?
+      str.puts
       str.puts Log.color(:magenta, "Inputs")
       str.puts
       str.puts SOPT.input_array_doc(inputs)
-      str.puts
+
+      inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
+        selects << [name, options[:select_options]] if options[:select_options]
+      end
     end
 
     deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
@@ -44,14 +46,21 @@ module Task
       deps.each do |dep_workflow,task_name,options|
         next if task_name.nil?
         task = dep_workflow.tasks[task_name]
+
         next if task.inputs.nil?
+
         inputs = task.inputs.reject{|name, _| seen.include? name }
         inputs = task.inputs.reject{|name, _| options.include? name }
         next unless inputs.any?
+        task.inputs.select{|name, _| inputs.include? name }.each do |name,_,_,_,options|
+          selects << [i, options[:select_options]] if options[:select_options]
+        end
+
         dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
         dep_inputs[dep] = inputs
       end
 
+      str.puts
       str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
       dep_inputs.each do |dep,inputs|
         str.puts
@@ -59,54 +68,31 @@ module Task
         str.puts
         str.puts SOPT.input_array_doc(inputs)
       end
-
-      #task_inputs = dep_inputs deps, workflow
-      #task_inputs.each do |task,new_inputs|
-      #  new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t| 
-      #    t.to_sym == :select and task.input_options[i][:select_options] 
-      #  end.each do |i,t| 
-      #    selects << [i, task.input_options[i][:select_options]] 
-      #  end
-
-      #  next if new_inputs.empty?
-
-      #  if task.workflow and task.workflow != workflow
-      #    puts "  #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
-      #  else
-      #    puts "  #{Log.color :yellow, task.name.to_s}:"
-      #  end
-      #  puts unless Log.compact
-      #  puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
-      #  puts unless Log.compact
-      #end
-      #puts
     end
 
     case
     when inputs && inputs.select{|name,type| type == :array }.any?
+      str.puts
       str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
       also works in addition to the others. You may use the '--array_separator' option
       the change this default. Whenever a file is specified it may also accept STDIN using
       the '-' character."))
-      str.puts
 
     when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
-      str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
       str.puts
+      str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
     end
 
     str.puts
     str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
-    str.puts
 
     if selects.any?
-      str.puts Log.color(:magenta, "Input select options")
       str.puts
+      str.puts Log.color(:magenta, "Input select options")
       selects.collect{|p| p}.uniq.each do |input,options|
+        str.puts 
         str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
-        str.puts unless Log.compact
       end
-      str.puts
     end
     str.rewind
     str.read
@@ -150,6 +136,7 @@ module Workflow
 
     dep_tree = {}
     task = self.tasks[task_name]
+    raise "TaskNotFound: #{task_name}" if task.nil?
     task.deps.each do |workflow, task, options|
       next if seen.include? dep
       seen << [workflow, task, options.merge(seen_options)]
@@ -249,7 +236,9 @@ module Workflow
 
     str.puts
 
-    if task.nil?
+    if tasks.nil?
+      str.puts Log.color(:title, "No tasks")
+    elsif task.nil?
 
       if self.documentation[:description] and not self.documentation[:description].empty?
         str.puts Misc.format_paragraph self.documentation[:description] 
@@ -284,7 +273,7 @@ module Workflow
 
         prov_string = prov_string(dep_tree(name))
         str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
-      end
+      end 
 
     else