scout-gear 10.7.0 → 10.7.2

data/lib/scout/tsv/util/process.rb

@@ -44,11 +44,14 @@ module TSV
   end
 
   def add_field(name = nil)
-    through do |key, values|
+    keys.each do |key|
+      values = self[key]
       new_values = yield(key, values)
       new_values = [new_values].compact if type == :double and not Array === new_values
 
       case
+      when type == :single
+        values = new_values
       when (values.nil? and (fields.nil? or fields.empty?))
         values = [new_values]
       when values.nil?  

data/lib/scout/tsv/util/select.rb

@@ -156,8 +156,14 @@ module TSV
       case
       when ((Array === method) and (key == :key or key_field == key))
         with_unnamed do
-          keys.each do |key|
-            new[key] = self[key] if invert ^ (method.include? key)
+          if invert
+            keys.each do |key|
+              new[key] = self[key] unless method.include?(key)
+            end
+          else
+            method.each do |key|
+              new[key] = self[key] if self.include?(key)
+            end
           end
         end
       when Array === method

data/lib/scout/tsv/util/sort.rb

@@ -1,4 +1,10 @@
+require 'scout/entity'
+
 module TSV
+  def prepare_entity(...)
+    Entity.prepare_entity(...)
+  end
+
   def sort_by(field = nil, just_keys = false, &block)
     field = :all if field.nil?
 
@@ -8,16 +14,18 @@ module TSV
       elems = []
       case type
       when :single
-        through :key, field do |key, field|
-          elems << [key, field]
+        through :key, field do |key, value|
+          elems << [key, value]
         end
       when :list, :flat
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, value|
+          v = field == :key ? key : value[0]
+          elems << [key, v]
         end
       when :double
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, value|
+          v = field == :key ? key : value[0]
+          elems << [key, v]
         end
       end
     end
@@ -81,16 +89,16 @@ module TSV
       elems = []
       case type
       when :single
-        through :key, field do |key, field|
-          elems << [key, field]
+        through :key, field do |key, value|
+          elems << [key, value]
         end
       when :list, :flat
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, values|
+          elems << [key, values[0]]
         end
       when :double
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, values|
+          elems << [key, values[0]]
         end
       end
     end
@@ -99,7 +107,7 @@ module TSV
       if fields == :all
         if just_keys
           keys = elems.sort_by{|key, value| key }.collect{|key, values| key}
-          keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
+          keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true))
         else
           elems.sort_by{|key, value| key }
         end
@@ -128,7 +136,7 @@ module TSV
         end
         if just_keys
           keys = sorted.collect{|key, value| key}
-          keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
+          keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
           keys
         else
           sorted.collect{|key, value| [key, self[key]]}
@@ -137,7 +145,7 @@ module TSV
     else
       if just_keys
         keys = elems.sort(&block).collect{|key, value| key}
-        keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
+        keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
         keys
       else
         elems.sort(&block).collect{|key, value| [key, self[key]]}

data/lib/scout/tsv/util.rb

@@ -32,6 +32,7 @@ module TSV
       text
     end
 
+    path = path.find if Path === path
     TmpFile.with_file(values.uniq * "\n", false) do |value_file|
       cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' | sed 's/HEADERNOMATCH//' |sort -u|cut -f 2  |sort|uniq -c|sed 's/^ *//;s/ /\t/'"
       begin
@@ -84,8 +85,9 @@ module TSV
 
   def each(*args, &block)
     if block_given?
+      actual_unnamed = @unnamed.nil? ? true : @unnamed
       super(*args) do |k,v|
-        NamedArray.setup(v, @fields) unless @unnamed || @type == :flat || ! (Array === v)
+        NamedArray.setup(v, @fields) unless actual_unnamed || @type == :flat || ! (Array === v)
         block.call(k, v)
       end
     else
@@ -105,7 +107,8 @@ module TSV
     end
   end
 
-  def with_unnamed(unnamed = true)
+  def with_unnamed(unnamed = nil)
+    unnamed = true if unnamed.nil?
     begin
       old_unnamed = @unnamed
       @unnamed = unnamed
@@ -177,4 +180,10 @@ Example:
     self.annotate(super(other))
   end
 
+  def merge_zip(other)
+    other.each do |k,v|
+      self.zip_new k, v
+    end
+  end
+
 end

data/lib/scout/tsv.rb

@@ -16,7 +16,8 @@ require_relative 'tsv/csv'
 
 module TSV
   extend Annotation
-  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer
+  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer, :entity_options
+
 
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
@@ -34,7 +35,6 @@ module TSV
     alias original_setup setup
 
     def setup(obj, *rest, &block)
-
       if rest.length == 1 && String === rest.first
         options = TSV.str2options(rest.first)
         if Array === obj
@@ -44,7 +44,7 @@ module TSV
                           when :single
                             nil
                           end
-          obj = Misc.array2hash(obj, default_value)
+          obj = IndiferentHash.array2hash(obj, default_value)
         end
         original_setup(obj, options, &block)
       else
@@ -57,7 +57,7 @@ module TSV
                           when :single
                             nil
                           end
-          obj = Misc.array2hash(obj, default_value)
+          obj = IndiferentHash.array2hash(obj, default_value)
         end
         original_setup(obj, *rest, &block)
       end
@@ -74,11 +74,17 @@ module TSV
   end
 
   def self.open(file, options = {})
-    grep, invert_grep, nocache, monitor, entity_options = IndiferentHash.process_options options, :grep, :invert_grep, :nocache, :monitor, :entity_options
+    grep, invert_grep, fixed_grep, nocache, monitor, entity_options, unnamed, field = IndiferentHash.process_options options, :grep, :invert_grep, :fixed_grep, :nocache, :monitor, :entity_options, :unnamed, :field
+
+    if field and options[:field].nil?
+      options[:fields] = [field]
+      options[:type] ||= :single
+    end
 
     persist_options = IndiferentHash.pull_keys options, :persist
     persist_options = IndiferentHash.add_defaults persist_options, prefix: "TSV", type: :HDB, persist: false
     persist_options[:data] ||= options[:data]
+    persist_options[:update] = options.delete(:update) if options.include?(:update)
 
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
 
@@ -113,14 +119,20 @@ module TSV
       end
 
       tsv = if TSV::Parser === file
-        TSV.parse(file, **options)
-      else
-        options[:tsv_invert_grep] ||= invert_grep if invert_grep
-        Open.open(file, grep: grep, invert_grep: invert_grep, nocache: nocache) do |f|
-          TSV.parse(f, **options)
-        end
+              TSV.parse(file, **options)
+            else
+              options[:tsv_invert_grep] ||= invert_grep if invert_grep
+              Open.open(file, grep: grep, invert_grep: invert_grep, fixed_grep: fixed_grep, nocache: nocache) do |f|
+                TSV.parse(f, **options)
+              end
+            end
+
+      if tsv.identifiers.nil? and Path === tsv.filename and tsv.filename.identifier_file_path
+        tsv.identifiers = tsv.filename.identifier_file_path.find if tsv.filename.identifier_file_path.exists?
       end
 
+      tsv.unnamed = unnamed unless unnamed.nil?
+
       tsv
     end
   end

data/lib/scout/workflow/definition.rb

@@ -140,8 +140,17 @@ module Workflow
     end
   end
 
-  FORGET_DEP_TASKS = ENV["SCOUT_FORGET_DEP_TASKS"] == "true"
-  REMOVE_DEP_TASKS = ENV["SCOUT_REMOVE_DEP_TASKS"] == "true"
+  FORGET_TASK_ALIAS = begin 
+                        %w(SCOUT_FORGET_TASK_ALIAS SCOUT_FORGET_DEP_TASKS RBBT_FORGET_DEP_TASKS).select do |var|
+                          ENV[var] == 'true'
+                        end.any?
+                      end
+  REMOVE_TASK_ALIAS = begin 
+                        remove = %w(SCOUT_REMOVE_TASK_ALIAS SCOUT_REMOVE_DEP_TASKS RBBT_REMOVE_DEP_TASKS).select do |var|
+                          ENV.include?(var) && ENV[var] != 'false'
+                        end.first
+                        remove.nil? ? false : remove
+                      end
   def task_alias(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
@@ -155,18 +164,24 @@ module Workflow
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join
-      raise dep.get_exception if dep.error?
+      raise dep.exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
-      set_info :type, dep.info[:type]
-      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
+      merge_info type: dep.info[:type], task_alias: true
+
+      forget = config :forget_task_alias, "forget_task_alias"
+      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_TASK_ALIAS if forget.nil?
+
       if forget
-        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
+        remove = config :remove_task_alias, "remove_task_alias"
+        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_TASK_ALIAS if remove.nil?
+
+        Log.medium "Forget task_alias (remove: #{remove}): #{short_path}"
 
         self.archive_deps
         self.copy_linked_files_dir
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
-        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
+        Open.link_dir dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
 
         if dep.overriden? 
           Open.link dep.path, self.tmp_path
@@ -179,9 +194,9 @@ module Workflow
           when 'recursive'
             (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
               next if d.overriden
-              d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
+              d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
             end
-            dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
+            dep.clean unless Scout::Config.get(:remove_dep, "task:#{dep.task_signature}", "task:#{dep.task_name}", "workflow:#{dep.workflow.name}", :default => true).to_s == 'false'
           end 
         end
       else

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -34,14 +34,16 @@ module Workflow
     end
 
     def self.job_rules(rules, job)
+      IndiferentHash.setup(rules)
       workflow = job.workflow.name
       task_name = job.task_name.to_s
       defaults = rules["defaults"] || {}
 
       return IndiferentHash.setup(defaults) unless rules[workflow]
-      return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
+      workflow_rules = IndiferentHash.setup(rules[workflow])
+      return IndiferentHash.setup(defaults) unless workflow_rules[task_name]
+      job_rules = IndiferentHash.setup(workflow_rules[task_name])
 
-      job_rules = IndiferentHash.setup(rules[workflow][task_name])
       defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
       job_rules
     end
@@ -85,7 +87,7 @@ module Workflow
         candidates = workload.
           select{|k,v| v.empty? }.
           collect{|k,v| k }.
-          reject{|k| k.done? || k.running? }
+          reject{|k| k.done? || k.running? || (k.error? && ! k.recoverable_error?) }
       else
         candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
           select{|k,v| v.empty? }.
@@ -107,7 +109,7 @@ module Workflow
 
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
-    def initialize(timer = 5, available_resources = {})
+    def initialize(timer = 5, available_resources = nil)
       available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
       @timer               = timer
       @available_resources = IndiferentHash.setup(available_resources)
@@ -192,7 +194,7 @@ module Workflow
         workload = Orchestrator.workload(jobs)
         all_jobs = workload.keys
 
-        all_jobs.each{|job| job.clean unless job.done? && job.updated? }
+        all_jobs.each{|job| job.clean unless (job.done? && job.updated?) || (job.error? && ! job.recoverable_error?) }
 
         top_level_jobs = jobs.collect{|job| job.path }
         failed_jobs = []
@@ -216,6 +218,7 @@ module Workflow
                     raise TryAgain
                   end
                 else
+                  Log.warn "Non-recoverable error in #{job.path}"
                   next
                 end
               ensure
@@ -253,7 +256,7 @@ module Workflow
     end
   end
 
-  def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors)
+  def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors, produce_timer = 5)
     jobs = [jobs] unless Array === jobs
     produce_list = []
     jobs.each do |job|
@@ -265,7 +268,7 @@ module Workflow
       end
     end
 
-    orchestrator = Orchestrator.new 0.1, cpus: produce_cpus.to_i
+    orchestrator = Orchestrator.new produce_timer, cpus: produce_cpus.to_i
     orchestrator.process({}, produce_list)
     produce_list
   end

data/lib/scout/workflow/exceptions.rb

@@ -0,0 +1 @@
+class TaskNotFound < StandardError; end

data/lib/scout/workflow/path.rb

@@ -0,0 +1,40 @@
+module Path
+  def self.step_file?(path)
+    return false unless path =~ /\.files(?:\/|$)/
+    parts = path.split("/")
+    job = parts.select{|p| p =~ /\.files$/}.first
+
+    if job
+      i = parts.index job
+      begin
+        workflow, task = parts.values_at i - 2, i - 1
+        _loaded = false
+        begin
+          Kernel.const_get(workflow)
+        rescue 
+          if ! _loaded
+            Workflow.require_workflow workflow 
+            _loaded = true
+            retry
+          end
+          raise $!
+        end
+        return parts[i-2..-1] * "/"
+      rescue
+        Log.exception $!
+      end
+    end
+
+    false
+  end
+
+  alias original_digest_str digest_str
+
+  def digest_str
+    if step_file = Path.step_file?(self)
+      "Step file: #{step_file}"
+    else
+      original_digest_str
+    end
+  end
+end

data/lib/scout/workflow/step/dependencies.rb

@@ -1,20 +1,26 @@
 class Step
   def rec_dependencies(connected = false, seen = [])
-    direct_deps = []
-    dependencies.each do |dep|
-      next if seen.include? dep.path
-      next if connected && dep.done? && dep.updated?
-      direct_deps << dep
-    end if dependencies
-    seen.concat direct_deps.collect{|d| d.path }
-    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
+    @rec_dependencies = {}
+    @rec_dependencies[connected] ||= begin
+                            direct_deps = []
+                            dependencies.each do |dep|
+                              next if seen.include? dep.path
+                              next if connected && dep.done? && dep.updated?
+                              direct_deps << dep
+                            end if dependencies
+                            seen.concat direct_deps.collect{|d| d.path }
+                            seen.uniq!
+                            direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
+                            direct_deps.uniq!
+                            direct_deps
+                          end
   end
 
   def recursive_inputs
-    recursive_inputs = NamedArray === @inputs ? @inputs.to_hash : {}
+    recursive_inputs = NamedArray === inputs ? inputs.to_hash : {}
     return recursive_inputs if dependencies.nil?
     dependencies.inject(recursive_inputs) do |acc,dep|
-      acc.merge(dep.recursive_inputs)
+      acc = dep.recursive_inputs.merge(acc)
     end
   end
 
@@ -41,6 +47,7 @@ class Step
       end
 
       next if dep.done?
+      next if dep.error? && ! dep.recoverable_error?
 
       if dep.dependencies
         dep.dependencies.each do |d|
@@ -72,9 +79,13 @@ class Step
   def run_dependencies
     all_dependencies.each do |dep| 
       next if dep.running? || dep.done?
+      next if dep.error? && ! dep.recoverable_error?
+
       compute_options = compute[dep.path] if compute
       compute_options = [] if compute_options.nil?
 
+      next if compute_options.include?(false)
+
       stream = compute_options.include?(:stream)
       stream = true unless ENV["SCOUT_EXPLICIT_STREAMING"] == 'true'
       stream = :no_load if compute_options.include?(:produce)
@@ -95,12 +106,25 @@ class Step
     all_dependencies.each{|dep| dep.abort if dep.running? } 
   end
 
-  def self.wait_for_jobs(jobs)
+  def self.wait_for_jobs(jobs, canfail=false)
     threads = []
     jobs.each do |job|
       threads << Thread.new do
         Thread.current.report_on_exception = false
-        job.join
+        begin
+          job.join
+        rescue Exception
+          case canfail
+          when TrueClass
+            next
+          else
+            if canfail === $!
+              next
+            else
+              raise $!
+            end
+          end
+        end
       end
     end
     threads.each do |t|

data/lib/scout/workflow/step/file.rb

@@ -7,6 +7,7 @@ class Step
                      else
                        Path.setup(dir)
                      end
+                     dir = dir.find
                      dir.pkgdir = self
                      dir
                    end
@@ -34,7 +35,7 @@ class Step
       begin
         realpath = Open.realpath(self.files_dir)
         Open.rm self.files_dir
-        Open.cp realpath, self.files_dir
+        Open.link_dir realpath, self.files_dir
       rescue
         Log.warn "Copy files_dir for #{self.workflow_short_path} failed: " + $!.message
       end

data/lib/scout/workflow/step/info.rb

@@ -10,7 +10,11 @@ class Step
   end
 
   def self.load_info(info_file)
-    info = Persist.load(info_file, SERIALIZER) || {}
+    info = begin
+             Persist.load(info_file, SERIALIZER) || {}
+           rescue
+             {status: :noinfo}
+           end
     IndiferentHash.setup(info)
   end
 
@@ -20,7 +24,7 @@ class Step
   end
 
   def save_info(info = nil)
-    Persist.save(info, info_file, SERIALIZER)
+    Persist.save(@info = info, info_file, SERIALIZER)
     @info_load_time = Time.now
   end
 
@@ -117,6 +121,7 @@ class Step
     if message.nil?
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
     else
+      message = Log.fingerprint(message).sub(/^'/,'').sub(/'$/,'')
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
     end
   end
@@ -139,6 +144,10 @@ class Step
     info[:messages]
   end
 
+  def message(message)
+    merge_info :message => message
+  end
+
   def status
     info[:status].tap{|s| s.nil? ? s : s.to_sym }
   end
@@ -152,7 +161,7 @@ class Step
   end
 
   def running?
-    ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
+    ! (done? && status == :done) && (info[:pid] && Misc.pid_alive?(info[:pid]))
   end
 
   def overriden?
@@ -161,7 +170,14 @@ class Step
   end
 
   def overriden_deps
-    rec_dependencies.select{|d| d.overriden? }
+    dependencies.select{|d| d.overriden? }
+  end
+
+  def recursive_overriden_deps
+    overriden_deps = dependencies.select{|d| 
+      d.overriden? 
+    }
+    (overriden_deps + overriden_deps.collect{|d| d.recursive_overriden_deps }).flatten.uniq
   end
 
   def exception

data/lib/scout/workflow/step/load.rb

@@ -4,9 +4,11 @@ class Step
     Path.setup(path) unless Path === path
     relocated = path.relocate
     return relocated if Open.exists?(relocated)
-    subpath = path.split("/")[-3..-1] * "/"
-    relocated = Path.setup("var/jobs")[subpath]
-    return relocated if Open.exists?(relocated)
+    if path.scan("/").length >= 2
+      subpath = path.split("/")[-3..-1] * "/"
+      relocated = Path.setup("var/jobs")[subpath]
+      return relocated if Open.exists?(relocated)
+    end
     path
   end
 

data/lib/scout/workflow/step/progress.rb

@@ -19,5 +19,11 @@ class Step
       bar
     end
   end
+
+  def traverse(obj, desc: nil , **kwargs, &block)
+    desc = "Processing #{self.short_path}" if desc.nil?
+    kwargs[:bar] = self.progress_bar(desc) unless kwargs.include?(:bar)
+    TSV.traverse obj, **kwargs, &block
+  end
 end
 

data/lib/scout/workflow/step/provenance.rb

@@ -62,7 +62,7 @@ class Step
           end
 
     if $inputs and $inputs.any? 
-      job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
+      job_inputs = Step.new(path).recursive_inputs.to_hash
       IndiferentHash.setup(job_inputs)
 
       $inputs.each do |input|

data/lib/scout/workflow/step/status.rb

@@ -20,17 +20,24 @@ class Step
     self.error? && ! (ScoutException === self.exception)
   end
 
-  def updated?
-    return false if self.error? && self.recoverable_error?
-    return true if self.done? && ! ENV["SCOUT_UPDATE"]
+  def newer_dependencies
     newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
+    newer += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| Path.newer?(self.path, dep.path) }
+    newer
+  end
+
+  def updated?
+    return false if self.error? && self.recoverable_error?
+    return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"]
+    newer = newer_dependencies
 
     Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
     newer.empty?
   end
 
   def clean
+    Log.debug "Cleaning job files: #{path}"
     @take_stream = nil 
     @result = nil
     @info = nil
@@ -55,7 +62,7 @@ class Step
   end
 
   def canfail?
-    @compute && @compute.include?(:canfail)
+    @compute && @compute[self.path] && @compute[self.path].include?(:canfail)
   end
 
   def started?