scout-gear 10.7.1 → 10.7.3

data/lib/scout/tsv.rb

@@ -16,7 +16,8 @@ require_relative 'tsv/csv'
 
 module TSV
   extend Annotation
-  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer
+  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer, :entity_options
+
 
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
@@ -34,7 +35,6 @@ module TSV
     alias original_setup setup
 
     def setup(obj, *rest, &block)
-
       if rest.length == 1 && String === rest.first
         options = TSV.str2options(rest.first)
         if Array === obj
@@ -44,7 +44,7 @@ module TSV
                           when :single
                             nil
                           end
-          obj = Misc.array2hash(obj, default_value)
+          obj = IndiferentHash.array2hash(obj, default_value)
         end
         original_setup(obj, options, &block)
       else
@@ -57,7 +57,7 @@ module TSV
                           when :single
                             nil
                           end
-          obj = Misc.array2hash(obj, default_value)
+          obj = IndiferentHash.array2hash(obj, default_value)
         end
         original_setup(obj, *rest, &block)
       end
@@ -74,11 +74,17 @@ module TSV
   end
 
   def self.open(file, options = {})
-    grep, invert_grep, nocache, monitor, entity_options = IndiferentHash.process_options options, :grep, :invert_grep, :nocache, :monitor, :entity_options
+    grep, invert_grep, fixed_grep, nocache, monitor, entity_options, unnamed, field = IndiferentHash.process_options options, :grep, :invert_grep, :fixed_grep, :nocache, :monitor, :entity_options, :unnamed, :field
+
+    if field and options[:field].nil?
+      options[:fields] = [field]
+      options[:type] ||= :single
+    end
 
     persist_options = IndiferentHash.pull_keys options, :persist
     persist_options = IndiferentHash.add_defaults persist_options, prefix: "TSV", type: :HDB, persist: false
     persist_options[:data] ||= options[:data]
+    persist_options[:update] = options.delete(:update) if options.include?(:update)
 
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
 
@@ -113,14 +119,22 @@ module TSV
       end
 
       tsv = if TSV::Parser === file
-        TSV.parse(file, **options)
-      else
-        options[:tsv_invert_grep] ||= invert_grep if invert_grep
-        Open.open(file, grep: grep, invert_grep: invert_grep, nocache: nocache) do |f|
-          TSV.parse(f, **options)
-        end
+              TSV.parse(file, **options)
+            else
+              options[:tsv_invert_grep] ||= invert_grep if invert_grep
+              Open.open(file, grep: grep, invert_grep: invert_grep, fixed_grep: fixed_grep, nocache: nocache) do |f|
+                TSV.parse(f, **options)
+              end
+            end
+
+      if tsv.identifiers.nil? and Path === tsv.filename and tsv.filename.identifier_file_path
+        tsv.identifiers = tsv.filename.identifier_file_path.find if tsv.filename.identifier_file_path.exists?
       end
 
+      tsv.unnamed = unnamed unless unnamed.nil?
+
+      tsv.entity_options = entity_options
+
       tsv
     end
   end

data/lib/scout/workflow/definition.rb

@@ -164,9 +164,9 @@ module Workflow
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join
-      raise dep.get_exception if dep.error?
+      raise dep.exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
-      set_info :type, dep.info[:type]
+      merge_info type: dep.info[:type], task_alias: true
 
       forget = config :forget_task_alias, "forget_task_alias"
       forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_TASK_ALIAS if forget.nil?
@@ -181,7 +181,7 @@ module Workflow
         self.copy_linked_files_dir
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
-        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
+        Open.link_dir dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
 
         if dep.overriden? 
           Open.link dep.path, self.tmp_path

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -34,14 +34,16 @@ module Workflow
     end
 
     def self.job_rules(rules, job)
+      IndiferentHash.setup(rules)
       workflow = job.workflow.name
       task_name = job.task_name.to_s
       defaults = rules["defaults"] || {}
 
       return IndiferentHash.setup(defaults) unless rules[workflow]
-      return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
+      workflow_rules = IndiferentHash.setup(rules[workflow])
+      return IndiferentHash.setup(defaults) unless workflow_rules[task_name]
+      job_rules = IndiferentHash.setup(workflow_rules[task_name])
 
-      job_rules = IndiferentHash.setup(rules[workflow][task_name])
       defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
       job_rules
     end
@@ -85,7 +87,7 @@ module Workflow
         candidates = workload.
           select{|k,v| v.empty? }.
           collect{|k,v| k }.
-          reject{|k| k.done? || k.running? }
+          reject{|k| k.done? || k.running? || (k.error? && ! k.recoverable_error?) }
       else
         candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
           select{|k,v| v.empty? }.
@@ -107,7 +109,7 @@ module Workflow
 
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
-    def initialize(timer = 5, available_resources = {})
+    def initialize(timer = 5, available_resources = nil)
       available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
       @timer               = timer
       @available_resources = IndiferentHash.setup(available_resources)
@@ -192,7 +194,7 @@ module Workflow
         workload = Orchestrator.workload(jobs)
         all_jobs = workload.keys
 
-        all_jobs.each{|job| job.clean unless job.done? && job.updated? }
+        all_jobs.each{|job| job.clean unless (job.done? && job.updated?) || (job.error? && ! job.recoverable_error?) }
 
         top_level_jobs = jobs.collect{|job| job.path }
         failed_jobs = []
@@ -216,6 +218,7 @@ module Workflow
                     raise TryAgain
                   end
                 else
+                  Log.warn "Non-recoverable error in #{job.path}"
                   next
                 end
               ensure

data/lib/scout/workflow/step/dependencies.rb

@@ -1,20 +1,26 @@
 class Step
   def rec_dependencies(connected = false, seen = [])
-    direct_deps = []
-    dependencies.each do |dep|
-      next if seen.include? dep.path
-      next if connected && dep.done? && dep.updated?
-      direct_deps << dep
-    end if dependencies
-    seen.concat direct_deps.collect{|d| d.path }
-    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
+    @rec_dependencies = {}
+    @rec_dependencies[connected] ||= begin
+                            direct_deps = []
+                            dependencies.each do |dep|
+                              next if seen.include? dep.path
+                              next if connected && dep.done? && dep.updated?
+                              direct_deps << dep
+                            end if dependencies
+                            seen.concat direct_deps.collect{|d| d.path }
+                            seen.uniq!
+                            direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
+                            direct_deps.uniq!
+                            direct_deps
+                          end
   end
 
   def recursive_inputs
     recursive_inputs = NamedArray === inputs ? inputs.to_hash : {}
     return recursive_inputs if dependencies.nil?
     dependencies.inject(recursive_inputs) do |acc,dep|
-      acc.merge(dep.recursive_inputs)
+      acc = dep.recursive_inputs.merge(acc)
     end
   end
 
@@ -41,6 +47,7 @@ class Step
       end
 
       next if dep.done?
+      next if dep.error? && ! dep.recoverable_error?
 
       if dep.dependencies
         dep.dependencies.each do |d|
@@ -72,9 +79,13 @@ class Step
   def run_dependencies
     all_dependencies.each do |dep| 
       next if dep.running? || dep.done?
+      next if dep.error? && ! dep.recoverable_error?
+
       compute_options = compute[dep.path] if compute
       compute_options = [] if compute_options.nil?
 
+      next if compute_options.include?(false)
+
       stream = compute_options.include?(:stream)
       stream = true unless ENV["SCOUT_EXPLICIT_STREAMING"] == 'true'
       stream = :no_load if compute_options.include?(:produce)
@@ -95,12 +106,25 @@ class Step
     all_dependencies.each{|dep| dep.abort if dep.running? } 
   end
 
-  def self.wait_for_jobs(jobs)
+  def self.wait_for_jobs(jobs, canfail=false)
     threads = []
     jobs.each do |job|
       threads << Thread.new do
         Thread.current.report_on_exception = false
-        job.join
+        begin
+          job.join
+        rescue Exception
+          case canfail
+          when TrueClass
+            next
+          else
+            if canfail === $!
+              next
+            else
+              raise $!
+            end
+          end
+        end
       end
     end
     threads.each do |t|

data/lib/scout/workflow/step/file.rb

@@ -7,6 +7,7 @@ class Step
                      else
                        Path.setup(dir)
                      end
+                     dir = dir.find
                      dir.pkgdir = self
                      dir
                    end
@@ -34,7 +35,7 @@ class Step
       begin
         realpath = Open.realpath(self.files_dir)
         Open.rm self.files_dir
-        Open.cp realpath, self.files_dir
+        Open.link_dir realpath, self.files_dir
       rescue
         Log.warn "Copy files_dir for #{self.workflow_short_path} failed: " + $!.message
       end

data/lib/scout/workflow/step/info.rb

@@ -24,7 +24,7 @@ class Step
   end
 
   def save_info(info = nil)
-    Persist.save(info, info_file, SERIALIZER)
+    Persist.save(@info = info, info_file, SERIALIZER)
     @info_load_time = Time.now
   end
 
@@ -50,6 +50,15 @@ class Step
     @info
   end
 
+  def pid
+    info[:pid]
+  end
+
+  def pid=(pid)
+    set_info :pid, pid
+  end
+
+
   def merge_info(new_info)
     info = self.info
     new_info.each do |key,value|
@@ -121,6 +130,7 @@ class Step
     if message.nil?
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
     else
+      message = Log.fingerprint(message.split("\n").first).sub(/^'/,'').sub(/'$/,'')
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
     end
   end
@@ -143,6 +153,10 @@ class Step
     info[:messages]
   end
 
+  def message(message)
+    merge_info :message => message
+  end
+
   def status
     info[:status].tap{|s| s.nil? ? s : s.to_sym }
   end
@@ -165,7 +179,14 @@ class Step
   end
 
   def overriden_deps
-    rec_dependencies.select{|d| d.overriden? }
+    dependencies.select{|d| d.overriden? }
+  end
+
+  def recursive_overriden_deps
+    overriden_deps = dependencies.select{|d| 
+      d.overriden? 
+    }
+    (overriden_deps + overriden_deps.collect{|d| d.recursive_overriden_deps }).flatten.uniq
   end
 
   def exception

data/lib/scout/workflow/step/load.rb

@@ -4,9 +4,11 @@ class Step
     Path.setup(path) unless Path === path
     relocated = path.relocate
     return relocated if Open.exists?(relocated)
-    subpath = path.split("/")[-3..-1] * "/"
-    relocated = Path.setup("var/jobs")[subpath]
-    return relocated if Open.exists?(relocated)
+    if path.scan("/").length >= 2
+      subpath = path.split("/")[-3..-1] * "/"
+      relocated = Path.setup("var/jobs")[subpath]
+      return relocated if Open.exists?(relocated)
+    end
     path
   end
 

data/lib/scout/workflow/step/progress.rb

@@ -19,5 +19,11 @@ class Step
       bar
     end
   end
+
+  def traverse(obj, desc: nil , **kwargs, &block)
+    desc = "Processing #{self.short_path}" if desc.nil?
+    kwargs[:bar] = self.progress_bar(desc) unless kwargs.include?(:bar)
+    TSV.traverse obj, **kwargs, &block
+  end
 end
 

data/lib/scout/workflow/step/provenance.rb

@@ -62,7 +62,7 @@ class Step
           end
 
     if $inputs and $inputs.any? 
-      job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
+      job_inputs = Step.new(path).recursive_inputs.to_hash
       IndiferentHash.setup(job_inputs)
 
       $inputs.each do |input|

data/lib/scout/workflow/step/status.rb

@@ -20,11 +20,17 @@ class Step
     self.error? && ! (ScoutException === self.exception)
   end
 
-  def updated?
-    return false if self.error? && self.recoverable_error?
-    return true if self.done? && ! ENV["SCOUT_UPDATE"]
+  def newer_dependencies
     newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
+    newer += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| Path.newer?(self.path, dep.path) }
+    newer
+  end
+
+  def updated?
+    return false if self.error? && self.recoverable_error?
+    return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"]
+    newer = newer_dependencies
 
     Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
     newer.empty?
@@ -56,7 +62,7 @@ class Step
   end
 
   def canfail?
-    @compute && @compute.include?(:canfail)
+    @compute && @compute[self.path] && @compute[self.path].include?(:canfail)
   end
 
   def started?

data/lib/scout/workflow/step.rb

@@ -138,18 +138,32 @@ class Step
 
   attr_reader :result
   def run(stream = false)
+    case stream
+    when TrueClass, :stream
+      no_load = :stream
+    when :no_load
+      no_load = true
+    else
+      no_load = false
+    end
+
+    if done?
+      if no_load
+        if no_load == :stream
+          return self.stream
+        else
+          return self.path
+        end
+      else
+        return @result || self.load
+      end
+    end
+
+
     return @result || self.load if done?
     prepare_dependencies
     begin
 
-      case stream
-      when TrueClass, :stream
-        no_load = :stream
-      when :no_load
-        no_load = true
-      else
-        no_load = false
-      end
 
       @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies, no_load: no_load) do
         input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
@@ -206,6 +220,7 @@ class Step
           end
 
           @result.abort_callback = proc do |exception|
+            Open.rm self.path
             if exception.nil? || Aborted === exception || Interrupt === exception
               merge_info :status => :aborted, :end => Time.now
             else
@@ -240,6 +255,7 @@ class Step
         run(noload)
       end
       join
+      exit! 0
     end
     Process.detach pid
     grace
@@ -350,6 +366,10 @@ class Step
 
   def step(task_name)
     task_name = task_name.to_sym
+    dependencies.each do |dep|
+      return dep if dep.task_name && dep.task_name.to_sym == task_name
+      return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
+    end
     dependencies.each do |dep|
       return dep if dep.task_name && dep.task_name.to_sym == task_name
       return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
@@ -359,12 +379,12 @@ class Step
     nil
   end
 
-  def short_path
-    Resource.identify @path
+  def identify_path
+    Resource.identify path
   end
 
   def digest_str
-    "Step: " + short_path
+    "Step: " + identify_path
   end
 
   def fingerprint
@@ -373,7 +393,7 @@ class Step
 
   def task_signature
     workflow_name = String === workflow ? workflow : workflow.name
-    [workflow, task_name] * "#"
+    [workflow_name, task_name] * "#"
   end
 
   def alias?

data/lib/scout/workflow/task/dependencies.rb

@@ -9,22 +9,33 @@ module Task
     load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
       task = step_options.delete(:task) if step_options.include?(:task)
       workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
-      id = step_options.delete(:id) if step_options.include?(:id)
-      id = step_options.delete(:jobname) if step_options.include?(:jobname)
+
+      step_id = id
+      step_id = step_options.delete(:jobname) if step_options.include?(:jobname)
+      step_id = nil if step_id == Task::DEFAULT_NAME
 
       step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
-      step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
+      step_inputs = IndiferentHash.add_defaults step_inputs.dup, definition_options
 
       resolved_inputs = {}
       step_inputs.each do |k,v|
         if Symbol === v
           input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || v
+          resolved_inputs[k] = if input_dep
+                                 input_dep
+                               elsif provided_inputs.include?(v) && self.inputs.collect(&:first).include?(v)
+                                 provided_inputs[v]
+                               elsif step_inputs.include?(v) && self.inputs.collect(&:first).include?(v)
+                                 step_inputs[v]
+                               else
+                                 v
+                               end
         else
           resolved_inputs[k] = v
         end
       end
-      job = workflow.job(task, id, resolved_inputs)
+
+      job = workflow.job(task, step_id, resolved_inputs)
       compute_options = definition_options[:compute] || []
       compute_options = [compute_options] unless Array === compute_options
       compute_options << :canfail if definition_options[:canfail]
@@ -33,26 +44,25 @@ module Task
       compute[job.path] = compute_options if compute_options.any?
 
       job.overriden = false if definition_options[:not_overriden]
+      job.compute = job.compute.nil? ? compute : job.compute.merge(compute)
 
       [job, step_inputs]
     end
 
     # Helper function
-    find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
+    filter_dep_non_default_inputs = proc do |dep,definition_options|
       dep_non_default_inputs = dep.non_default_inputs
-      dep_non_default_inputs.select do |name|
-        step_inputs.include?(name)  
-      end
       dep_non_default_inputs.reject! do |name|
-        definition_options.include?(name) && 
-          (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
+        definition_options.include?(name)
       end
 
       dep_non_default_inputs
     end
 
     deps.each do |workflow,task,definition_options,block=nil|
-      definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
+      definition_options = definition_options.nil? ? {} : definition_options.dup
+
+      dep_id = definition_options.include?(:jobname) ? definition_options.delete(:jobname) : id
 
       if provided_inputs.include?(overriden = [workflow.name, task] * "#")
         dep = provided_inputs[overriden]
@@ -66,51 +76,50 @@ module Task
         next
       end
 
-      definition_options ||= {}
 
       if block
-        fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
-        self.inputs.each do |name,type,desc,value|
+        fixed_provided_inputs = self.assign_inputs(provided_inputs, dep_id).first.to_hash
+        self.inputs.each do |name,type,desc,value,options|
           fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
         end
         fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
         block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
 
-        res = block.call id, block_options, dependencies
+        res = block.call dep_id, block_options, dependencies
 
         case res
         when Step
           dep = res
           dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
+          dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
           non_default_inputs.concat(dep_non_default_inputs)
         when Hash
           step_options = block_options.merge(res)
-          dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+          dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options.dup, block_options, dependencies)
           dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+          dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
           non_default_inputs.concat(dep_non_default_inputs)
         when Array
           res.each do |_res|
             if Hash === _res
               step_options = block_options.merge(_res)
-              dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+              dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options.dup, block_options, dependencies)
               dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+              dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
               non_default_inputs.concat(dep_non_default_inputs)
             else
               dep = _res
               dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
+              dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
               non_default_inputs.concat(dep_non_default_inputs)
             end
           end
         end
       else
         step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
-        dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
+        dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options, definition_options, dependencies)
         dependencies << dep
-        dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+        dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
         non_default_inputs.concat(dep_non_default_inputs)
       end
     end