scout-gear 10.7.1 → 10.7.2

data/lib/scout/tsv/util/sort.rb

@@ -1,4 +1,10 @@
+require 'scout/entity'
+
 module TSV
+  def prepare_entity(...)
+    Entity.prepare_entity(...)
+  end
+
   def sort_by(field = nil, just_keys = false, &block)
     field = :all if field.nil?
 
@@ -8,16 +14,18 @@ module TSV
       elems = []
       case type
       when :single
-        through :key, field do |key, field|
-          elems << [key, field]
+        through :key, field do |key, value|
+          elems << [key, value]
         end
       when :list, :flat
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, value|
+          v = field == :key ? key : value[0]
+          elems << [key, v]
         end
       when :double
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, value|
+          v = field == :key ? key : value[0]
+          elems << [key, v]
         end
       end
     end
@@ -81,16 +89,16 @@ module TSV
       elems = []
       case type
       when :single
-        through :key, field do |key, field|
-          elems << [key, field]
+        through :key, field do |key, value|
+          elems << [key, value]
         end
       when :list, :flat
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, values|
+          elems << [key, values[0]]
         end
       when :double
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, values|
+          elems << [key, values[0]]
         end
       end
     end
@@ -99,7 +107,7 @@ module TSV
       if fields == :all
         if just_keys
           keys = elems.sort_by{|key, value| key }.collect{|key, values| key}
-          keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
+          keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true))
         else
           elems.sort_by{|key, value| key }
         end
@@ -128,7 +136,7 @@ module TSV
         end
         if just_keys
           keys = sorted.collect{|key, value| key}
-          keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
+          keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
           keys
         else
           sorted.collect{|key, value| [key, self[key]]}
@@ -137,7 +145,7 @@ module TSV
     else
       if just_keys
         keys = elems.sort(&block).collect{|key, value| key}
-        keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
+        keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
         keys
       else
         elems.sort(&block).collect{|key, value| [key, self[key]]}

data/lib/scout/tsv/util.rb

@@ -32,6 +32,7 @@ module TSV
       text
     end
 
+    path = path.find if Path === path
     TmpFile.with_file(values.uniq * "\n", false) do |value_file|
       cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' | sed 's/HEADERNOMATCH//' |sort -u|cut -f 2  |sort|uniq -c|sed 's/^ *//;s/ /\t/'"
       begin
@@ -84,8 +85,9 @@ module TSV
 
   def each(*args, &block)
     if block_given?
+      actual_unnamed = @unnamed.nil? ? true : @unnamed
       super(*args) do |k,v|
-        NamedArray.setup(v, @fields) unless @unnamed || @type == :flat || ! (Array === v)
+        NamedArray.setup(v, @fields) unless actual_unnamed || @type == :flat || ! (Array === v)
         block.call(k, v)
       end
     else
@@ -105,7 +107,8 @@ module TSV
     end
   end
 
-  def with_unnamed(unnamed = true)
+  def with_unnamed(unnamed = nil)
+    unnamed = true if unnamed.nil?
     begin
       old_unnamed = @unnamed
       @unnamed = unnamed
@@ -177,4 +180,10 @@ Example:
     self.annotate(super(other))
   end
 
+  def merge_zip(other)
+    other.each do |k,v|
+      self.zip_new k, v
+    end
+  end
+
 end

data/lib/scout/tsv.rb

@@ -16,7 +16,8 @@ require_relative 'tsv/csv'
 
 module TSV
   extend Annotation
-  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer
+  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer, :entity_options
+
 
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
@@ -34,7 +35,6 @@ module TSV
     alias original_setup setup
 
     def setup(obj, *rest, &block)
-
       if rest.length == 1 && String === rest.first
         options = TSV.str2options(rest.first)
         if Array === obj
@@ -44,7 +44,7 @@ module TSV
                           when :single
                             nil
                           end
-          obj = Misc.array2hash(obj, default_value)
+          obj = IndiferentHash.array2hash(obj, default_value)
         end
         original_setup(obj, options, &block)
       else
@@ -57,7 +57,7 @@ module TSV
                           when :single
                             nil
                           end
-          obj = Misc.array2hash(obj, default_value)
+          obj = IndiferentHash.array2hash(obj, default_value)
         end
         original_setup(obj, *rest, &block)
       end
@@ -74,11 +74,17 @@ module TSV
   end
 
   def self.open(file, options = {})
-    grep, invert_grep, nocache, monitor, entity_options = IndiferentHash.process_options options, :grep, :invert_grep, :nocache, :monitor, :entity_options
+    grep, invert_grep, fixed_grep, nocache, monitor, entity_options, unnamed, field = IndiferentHash.process_options options, :grep, :invert_grep, :fixed_grep, :nocache, :monitor, :entity_options, :unnamed, :field
+
+    if field and options[:field].nil?
+      options[:fields] = [field]
+      options[:type] ||= :single
+    end
 
     persist_options = IndiferentHash.pull_keys options, :persist
     persist_options = IndiferentHash.add_defaults persist_options, prefix: "TSV", type: :HDB, persist: false
     persist_options[:data] ||= options[:data]
+    persist_options[:update] = options.delete(:update) if options.include?(:update)
 
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
 
@@ -113,14 +119,20 @@ module TSV
       end
 
       tsv = if TSV::Parser === file
-        TSV.parse(file, **options)
-      else
-        options[:tsv_invert_grep] ||= invert_grep if invert_grep
-        Open.open(file, grep: grep, invert_grep: invert_grep, nocache: nocache) do |f|
-          TSV.parse(f, **options)
-        end
+              TSV.parse(file, **options)
+            else
+              options[:tsv_invert_grep] ||= invert_grep if invert_grep
+              Open.open(file, grep: grep, invert_grep: invert_grep, fixed_grep: fixed_grep, nocache: nocache) do |f|
+                TSV.parse(f, **options)
+              end
+            end
+
+      if tsv.identifiers.nil? and Path === tsv.filename and tsv.filename.identifier_file_path
+        tsv.identifiers = tsv.filename.identifier_file_path.find if tsv.filename.identifier_file_path.exists?
       end
 
+      tsv.unnamed = unnamed unless unnamed.nil?
+
       tsv
     end
   end

data/lib/scout/workflow/definition.rb

@@ -164,9 +164,9 @@ module Workflow
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join
-      raise dep.get_exception if dep.error?
+      raise dep.exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
-      set_info :type, dep.info[:type]
+      merge_info type: dep.info[:type], task_alias: true
 
       forget = config :forget_task_alias, "forget_task_alias"
       forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_TASK_ALIAS if forget.nil?
@@ -181,7 +181,7 @@ module Workflow
         self.copy_linked_files_dir
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
-        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
+        Open.link_dir dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
 
         if dep.overriden? 
           Open.link dep.path, self.tmp_path

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -34,14 +34,16 @@ module Workflow
     end
 
     def self.job_rules(rules, job)
+      IndiferentHash.setup(rules)
       workflow = job.workflow.name
       task_name = job.task_name.to_s
       defaults = rules["defaults"] || {}
 
       return IndiferentHash.setup(defaults) unless rules[workflow]
-      return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
+      workflow_rules = IndiferentHash.setup(rules[workflow])
+      return IndiferentHash.setup(defaults) unless workflow_rules[task_name]
+      job_rules = IndiferentHash.setup(workflow_rules[task_name])
 
-      job_rules = IndiferentHash.setup(rules[workflow][task_name])
       defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
       job_rules
     end
@@ -85,7 +87,7 @@ module Workflow
         candidates = workload.
           select{|k,v| v.empty? }.
           collect{|k,v| k }.
-          reject{|k| k.done? || k.running? }
+          reject{|k| k.done? || k.running? || (k.error? && ! k.recoverable_error?) }
       else
         candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
           select{|k,v| v.empty? }.
@@ -107,7 +109,7 @@ module Workflow
 
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
-    def initialize(timer = 5, available_resources = {})
+    def initialize(timer = 5, available_resources = nil)
       available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
       @timer               = timer
       @available_resources = IndiferentHash.setup(available_resources)
@@ -192,7 +194,7 @@ module Workflow
         workload = Orchestrator.workload(jobs)
         all_jobs = workload.keys
 
-        all_jobs.each{|job| job.clean unless job.done? && job.updated? }
+        all_jobs.each{|job| job.clean unless (job.done? && job.updated?) || (job.error? && ! job.recoverable_error?) }
 
         top_level_jobs = jobs.collect{|job| job.path }
         failed_jobs = []
@@ -216,6 +218,7 @@ module Workflow
                     raise TryAgain
                   end
                 else
+                  Log.warn "Non-recoverable error in #{job.path}"
                   next
                 end
               ensure

data/lib/scout/workflow/step/dependencies.rb

@@ -1,20 +1,26 @@
 class Step
   def rec_dependencies(connected = false, seen = [])
-    direct_deps = []
-    dependencies.each do |dep|
-      next if seen.include? dep.path
-      next if connected && dep.done? && dep.updated?
-      direct_deps << dep
-    end if dependencies
-    seen.concat direct_deps.collect{|d| d.path }
-    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
+    @rec_dependencies = {}
+    @rec_dependencies[connected] ||= begin
+                            direct_deps = []
+                            dependencies.each do |dep|
+                              next if seen.include? dep.path
+                              next if connected && dep.done? && dep.updated?
+                              direct_deps << dep
+                            end if dependencies
+                            seen.concat direct_deps.collect{|d| d.path }
+                            seen.uniq!
+                            direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
+                            direct_deps.uniq!
+                            direct_deps
+                          end
   end
 
   def recursive_inputs
     recursive_inputs = NamedArray === inputs ? inputs.to_hash : {}
     return recursive_inputs if dependencies.nil?
     dependencies.inject(recursive_inputs) do |acc,dep|
-      acc.merge(dep.recursive_inputs)
+      acc = dep.recursive_inputs.merge(acc)
     end
   end
 
@@ -41,6 +47,7 @@ class Step
       end
 
       next if dep.done?
+      next if dep.error? && ! dep.recoverable_error?
 
       if dep.dependencies
         dep.dependencies.each do |d|
@@ -72,9 +79,13 @@ class Step
   def run_dependencies
     all_dependencies.each do |dep| 
       next if dep.running? || dep.done?
+      next if dep.error? && ! dep.recoverable_error?
+
       compute_options = compute[dep.path] if compute
       compute_options = [] if compute_options.nil?
 
+      next if compute_options.include?(false)
+
       stream = compute_options.include?(:stream)
       stream = true unless ENV["SCOUT_EXPLICIT_STREAMING"] == 'true'
       stream = :no_load if compute_options.include?(:produce)
@@ -95,12 +106,25 @@ class Step
     all_dependencies.each{|dep| dep.abort if dep.running? } 
   end
 
-  def self.wait_for_jobs(jobs)
+  def self.wait_for_jobs(jobs, canfail=false)
     threads = []
     jobs.each do |job|
       threads << Thread.new do
         Thread.current.report_on_exception = false
-        job.join
+        begin
+          job.join
+        rescue Exception
+          case canfail
+          when TrueClass
+            next
+          else
+            if canfail === $!
+              next
+            else
+              raise $!
+            end
+          end
+        end
       end
     end
     threads.each do |t|

data/lib/scout/workflow/step/file.rb

@@ -7,6 +7,7 @@ class Step
                      else
                        Path.setup(dir)
                      end
+                     dir = dir.find
                      dir.pkgdir = self
                      dir
                    end
@@ -34,7 +35,7 @@ class Step
       begin
         realpath = Open.realpath(self.files_dir)
         Open.rm self.files_dir
-        Open.cp realpath, self.files_dir
+        Open.link_dir realpath, self.files_dir
       rescue
         Log.warn "Copy files_dir for #{self.workflow_short_path} failed: " + $!.message
       end

data/lib/scout/workflow/step/info.rb

@@ -24,7 +24,7 @@ class Step
   end
 
   def save_info(info = nil)
-    Persist.save(info, info_file, SERIALIZER)
+    Persist.save(@info = info, info_file, SERIALIZER)
     @info_load_time = Time.now
   end
 
@@ -121,6 +121,7 @@ class Step
     if message.nil?
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
     else
+      message = Log.fingerprint(message).sub(/^'/,'').sub(/'$/,'')
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
     end
   end
@@ -143,6 +144,10 @@ class Step
     info[:messages]
   end
 
+  def message(message)
+    merge_info :message => message
+  end
+
   def status
     info[:status].tap{|s| s.nil? ? s : s.to_sym }
   end
@@ -165,7 +170,14 @@ class Step
   end
 
   def overriden_deps
-    rec_dependencies.select{|d| d.overriden? }
+    dependencies.select{|d| d.overriden? }
+  end
+
+  def recursive_overriden_deps
+    overriden_deps = dependencies.select{|d| 
+      d.overriden? 
+    }
+    (overriden_deps + overriden_deps.collect{|d| d.recursive_overriden_deps }).flatten.uniq
   end
 
   def exception

data/lib/scout/workflow/step/load.rb

@@ -4,9 +4,11 @@ class Step
     Path.setup(path) unless Path === path
     relocated = path.relocate
     return relocated if Open.exists?(relocated)
-    subpath = path.split("/")[-3..-1] * "/"
-    relocated = Path.setup("var/jobs")[subpath]
-    return relocated if Open.exists?(relocated)
+    if path.scan("/").length >= 2
+      subpath = path.split("/")[-3..-1] * "/"
+      relocated = Path.setup("var/jobs")[subpath]
+      return relocated if Open.exists?(relocated)
+    end
     path
   end
 

data/lib/scout/workflow/step/progress.rb

@@ -19,5 +19,11 @@ class Step
       bar
     end
   end
+
+  def traverse(obj, desc: nil , **kwargs, &block)
+    desc = "Processing #{self.short_path}" if desc.nil?
+    kwargs[:bar] = self.progress_bar(desc) unless kwargs.include?(:bar)
+    TSV.traverse obj, **kwargs, &block
+  end
 end
 

data/lib/scout/workflow/step/provenance.rb

@@ -62,7 +62,7 @@ class Step
           end
 
     if $inputs and $inputs.any? 
-      job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
+      job_inputs = Step.new(path).recursive_inputs.to_hash
       IndiferentHash.setup(job_inputs)
 
       $inputs.each do |input|

data/lib/scout/workflow/step/status.rb

@@ -20,11 +20,17 @@ class Step
     self.error? && ! (ScoutException === self.exception)
   end
 
-  def updated?
-    return false if self.error? && self.recoverable_error?
-    return true if self.done? && ! ENV["SCOUT_UPDATE"]
+  def newer_dependencies
     newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
+    newer += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| Path.newer?(self.path, dep.path) }
+    newer
+  end
+
+  def updated?
+    return false if self.error? && self.recoverable_error?
+    return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"]
+    newer = newer_dependencies
 
     Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
     newer.empty?
@@ -56,7 +62,7 @@ class Step
   end
 
   def canfail?
-    @compute && @compute.include?(:canfail)
+    @compute && @compute[self.path] && @compute[self.path].include?(:canfail)
   end
 
   def started?

data/lib/scout/workflow/step.rb

@@ -138,18 +138,32 @@ class Step
 
   attr_reader :result
   def run(stream = false)
+    case stream
+    when TrueClass, :stream
+      no_load = :stream
+    when :no_load
+      no_load = true
+    else
+      no_load = false
+    end
+
+    if done?
+      if no_load
+        if no_load == :stream
+          return self.stream
+        else
+          return self.path
+        end
+      else
+        return @result || self.load
+      end
+    end
+
+
     return @result || self.load if done?
     prepare_dependencies
     begin
 
-      case stream
-      when TrueClass, :stream
-        no_load = :stream
-      when :no_load
-        no_load = true
-      else
-        no_load = false
-      end
 
       @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies, no_load: no_load) do
         input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
@@ -206,6 +220,7 @@ class Step
           end
 
           @result.abort_callback = proc do |exception|
+            Open.rm self.path
             if exception.nil? || Aborted === exception || Interrupt === exception
               merge_info :status => :aborted, :end => Time.now
             else
@@ -240,6 +255,7 @@ class Step
         run(noload)
       end
       join
+      exit! 0
     end
     Process.detach pid
     grace
@@ -350,6 +366,10 @@ class Step
 
   def step(task_name)
     task_name = task_name.to_sym
+    dependencies.each do |dep|
+      return dep if dep.task_name && dep.task_name.to_sym == task_name
+      return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
+    end
     dependencies.each do |dep|
       return dep if dep.task_name && dep.task_name.to_sym == task_name
       return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
@@ -359,12 +379,12 @@ class Step
     nil
   end
 
-  def short_path
-    Resource.identify @path
+  def identify_path
+    Resource.identify path
   end
 
   def digest_str
-    "Step: " + short_path
+    "Step: " + identify_path
   end
 
   def fingerprint
@@ -373,7 +393,7 @@ class Step
 
   def task_signature
     workflow_name = String === workflow ? workflow : workflow.name
-    [workflow, task_name] * "#"
+    [workflow_name, task_name] * "#"
   end
 
   def alias?