scout-gear 8.0.0 → 9.0.0

data/lib/scout/tsv.rb

@@ -10,10 +10,11 @@ require_relative 'tsv/traverse'
 require_relative 'tsv/open'
 require_relative 'tsv/attach'
 require_relative 'tsv/change_id'
+require_relative 'tsv/stream'
 
 module TSV
   extend MetaExtension
-  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed, :identifiers
+  extension_attr :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers
 
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
@@ -40,7 +41,13 @@ module TSV
       data = filename ? ScoutCabinet.open(filename, true, type) : nil
       options[:data] = data if data
       options[:filename] = file
-      Log.debug "TSV open #{Log.fingerprint file}"
+
+      if data
+        Log.debug "TSV open #{Log.fingerprint file} into #{Log.fingerprint data}"
+      else
+        Log.debug "TSV open #{Log.fingerprint file}"
+      end
+
       Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
         TSV.parse(f, **options)
       end

data/lib/scout/work_queue/worker.rb

@@ -27,10 +27,10 @@ class WorkQueue
         rescue Interrupt
         rescue Exception
           output.write WorkerException.new($!, Process.pid)
-          Process.exit! -1
+          exit -1
         ensure
         end
-        Process.exit! 0
+        exit 0
       end
     end
 

data/lib/scout/work_queue.rb

@@ -1,10 +1,12 @@
 require_relative 'work_queue/socket'
 require_relative 'work_queue/worker'
+require 'timeout'
 
 class WorkQueue
   attr_accessor :workers, :worker_proc, :callback
 
   def initialize(workers = 0, &block)
+    workers = workers.to_i if String === workers
     @input = WorkQueue::Socket.new
     @output = WorkQueue::Socket.new
     @workers = workers.times.collect{ Worker.new }
@@ -38,7 +40,7 @@ class WorkQueue
     @worker_mutex.synchronize do
       worker = @workers.index{|w| w.pid == pid}
       if worker
-        Log.debug "Removed worker #{pid}"
+        Log.low "Removed worker #{pid}"
         @workers.delete_at(worker)
         @removed_workers << pid
       end
@@ -88,19 +90,32 @@ class WorkQueue
 
     Thread.pass until @reader["name"]
 
+    Thread.pass until @worker_mutex.synchronize{ @workers.select{|w| w.pid.nil? }.empty? }
+
     @waiter = Thread.new do
-      begin
-        Thread.current.report_on_exception = false
-        Thread.current["name"] = "Worker waiter #{Process.pid}"
-        while true
-          pid = Process.wait
-          remove_worker(pid)
-          break if @worker_mutex.synchronize{ @workers.empty? }
+      Thread.current.report_on_exception = false
+      Thread.current["name"] = "Worker waiter #{Process.pid}"
+      while true
+        break if @worker_mutex.synchronize{ @workers.empty? }
+        begin
+          Timeout.timeout(1) do
+            begin
+              pid, status = Process.wait2
+              remove_worker(pid) if pid
+            rescue Exception
+              Log.exception $!
+            end
+          end
+        rescue Timeout::Error
+          pids = @worker_mutex.synchronize{ @workers.collect{|w| w.pid } }
+          pids.each do |p|
+            pid, status = Process.wait2 p, Process::WNOHANG
+            remove_worker(pid) if pid
+          end
         end
       end
     end
 
-    Thread.pass until @worker_mutex.synchronize{ @workers.select{|w| w.pid.nil? }.empty? }
     Thread.pass until @waiter["name"]
   end
 
@@ -128,8 +143,18 @@ class WorkQueue
     end
   end
 
-  def join
-    @waiter.join if @waiter
-    @reader.join if @reader
+  def clean
+    @waiter.join if @waiter 
+    @input.clean
+    @output.clean
+  end
+
+  def join(clean = true)
+    begin
+      @waiter.join if @waiter
+      @reader.join if @reader
+    ensure
+      self.clean if clean
+    end
   end
 end

data/lib/scout/workflow/definition.rb

@@ -50,7 +50,7 @@ module Workflow
 
   def directory=(directory)
     @directory = directory
-    @tasks.each{|name,d| d.directory = directory[name] } if @tasks
+    @tasks.each{|name,d| d.directory = Path === directory ? directory[name] : File.join(directory, name.to_s) } if @tasks
   end
 
   def annotate_next_task(type, obj)
@@ -103,6 +103,7 @@ module Workflow
   def task(name_and_type, &block)
     name, type = name_and_type.collect.first
     @tasks ||= IndiferentHash.setup({})
+    block = self.method(name) if block.nil?
     begin
       @annotate_next_task ||= {}
       @annotate_next_task[:extension] ||=  

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -0,0 +1,254 @@
+module Workflow
+  class Orchestrator
+
+    def self.job_workload(job)
+      workload = {job => []}
+      return workload if job.done? && job.updated?
+
+      job.dependencies.each do |dep|
+        next if dep.done? && dep.updated?
+        workload.merge!(job_workload(dep))
+        workload[job] += workload[dep]
+        workload[job] << dep
+        workload[job].uniq!
+      end
+
+      job.input_dependencies.each do |dep|
+        next if dep.done? && dep.updated?
+        workload.merge!(job_workload(dep))
+        workload[job] += workload[dep]
+        workload[job] << dep
+        workload[job].uniq!
+      end
+
+      workload
+    end
+
+    def self.workload(jobs)
+      jobs.inject({}) do |acc,job| 
+        Orchestrator.job_workload(job).each do |j,d|
+          acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
+        end
+        acc
+      end
+    end
+
+    def self.job_rules(rules, job)
+      workflow = job.workflow.to_s
+      task_name = job.task_name.to_s
+      defaults = rules["defaults"] || {}
+
+      return IndiferentHash.setup(defaults) unless rules[workflow]
+      return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
+
+      job_rules = IndiferentHash.setup(rules[workflow][task_name])
+      defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
+      job_rules
+    end
+
+    def self.purge_duplicates(candidates)
+      seen = Set.new
+      candidates.select do |job|
+        if seen.include? job.path
+          false
+        else
+          seen << job.path
+          true
+        end
+      end
+    end
+
+    def self.job_resources(rules, job)
+      resources = (job_rules(rules, job) || {})["resources"] || {}
+
+      IndiferentHash.setup(resources)
+
+      default_resources = rules["default_resources"] 
+      default_resources ||= rules["defaults"]["resources"] if rules["defaults"]
+      default_resources ||= {}
+
+      default_resources.each{|k,v| resources[k] ||= v } if default_resources
+
+      resources = {:cpus => 1} if resources.empty?
+      resources
+    end
+
+    def self.sort_candidates(candidates, rules)
+      seen = Set.new
+      candidates.sort_by do |job|
+        - job_resources(rules, job).values.inject(0){|acc,e| acc += e}
+      end
+    end
+
+    def self.candidates(workload, rules)
+      if rules.empty?
+        candidates = workload.
+          select{|k,v| v.empty? }.
+          collect{|k,v| k }.
+          reject{|k| k.done? || k.running? }
+      else
+        candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
+          select{|k,v| v.empty? }.
+          collect{|k,v| k }.
+          reject{|k| k.done? || k.running? }
+      end
+
+      #top_level = workload.keys - workload.values.flatten
+
+      candidates = purge_duplicates candidates
+      candidates = sort_candidates candidates, rules
+
+      candidates
+    end
+
+    def self.process(*args)
+      self.new.process(*args)
+    end
+
+    attr_accessor :available_resources, :resources_requested, :resources_used, :timer
+
+    def initialize(timer = 5, available_resources = {})
+      available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
+      @timer               = timer
+      @available_resources = IndiferentHash.setup(available_resources)
+      @resources_requested = IndiferentHash.setup({})
+      @resources_used      = IndiferentHash.setup({})
+    end
+
+    def release_resources(job)
+      if resources_used[job]
+        Log.debug "Orchestrator releasing resouces from #{job.path}"
+        resources_used[job].each do |resource,value| 
+          next if resource == 'size'
+          resources_requested[resource] -= value.to_i
+        end
+        resources_used.delete job
+      end
+    end
+
+    def check_resources(rules, job)
+      resources = Orchestrator.job_resources(rules, job)
+
+      limit_resources = resources.select{|resource,value| available_resources[resource] && ((resources_requested[resource] || 0) + value) > available_resources[resource]  }.collect{|resource,v| resource }
+      if limit_resources.any?
+        Log.debug "Orchestrator waiting on #{job.path} due to #{limit_resources * ", "}"
+      else
+
+        resources_used[job] = resources
+        resources.each do |resource,value| 
+          resources_requested[resource] ||= 0
+          resources_requested[resource] += value.to_i
+        end
+        Log.low "Orchestrator producing #{job.path} with resources #{resources}"
+
+        return yield
+      end
+    end
+
+    def run_with_rules(rules, job)
+      job_rules = Orchestrator.job_rules(rules, job)
+
+      Scout::Config.with_config do 
+        job_rules[:config_keys].each do |config|
+          Scout::Config.process_config config
+        end if job_rules && job_rules[:config_keys]
+
+        log = job_rules[:log] if job_rules 
+        log = Log.severity if log.nil?
+        Log.with_severity log do
+          job.fork
+        end
+      end
+    end
+
+    def erase_job_dependencies(job, rules, all_jobs, top_level_jobs)
+      job.dependencies.each do |dep|
+        next if top_level_jobs.include? dep.path
+        next unless Orchestrator.job_rules(rules, dep)["erase"].to_s == 'true'
+
+        dep_path = dep.path
+        parents = all_jobs.select do |parent| 
+          paths = parent.info[:dependencies].nil? ? parent.dependencies.collect{|d| d.path } : parent.info[:dependencies].collect{|d| d.last }
+          paths.include? dep_path
+        end
+
+        next unless parents.reject{|parent| parent.done? }.empty?
+
+        parents.each do |parent|
+          Log.high "Erasing #{dep.path} from #{parent.path}"
+          parent.archive_deps
+          parent.copy_files_dir
+          parent.dependencies = parent.dependencies - [dep]
+        end
+        dep.clean
+      end
+    end
+
+    def process(rules, jobs = nil)
+      jobs, rules = rules, {} if jobs.nil?
+      jobs = [jobs] if Step === jobs
+      begin
+
+        workload = Orchestrator.workload(jobs)
+        all_jobs = workload.keys
+
+        all_jobs.each{|job| job.clean unless job.done? && job.updated? }
+
+        top_level_jobs = jobs.collect{|job| job.path }
+        failed_jobs = []
+        while workload.any? 
+
+          candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
+          candidates.uniq!
+          raise "No candidates and no running jobs" if candidates.empty?
+
+          candidates.each do |job|
+            case 
+            when (job.error? || job.aborted?)
+              begin
+                if job.recoverable_error? 
+                  if failed_jobs.include?(job)
+                    Log.warn "Failed twice #{job.path} with recoverable error"
+                    next
+                  else
+                    failed_jobs << job
+                    job.clean
+                    raise TryAgain
+                  end
+                else
+                  next
+                end
+              ensure
+                Log.warn "Releases resources from failed job: #{job.path}"
+                release_resources(job)
+              end
+            when job.done?
+              Log.debug "Orchestrator done #{job.path}"
+              release_resources(job)
+              erase_job_dependencies(job, rules, all_jobs, top_level_jobs)
+
+            when job.running?
+              next
+
+            else
+              check_resources(rules, job) do
+                run_with_rules(rules, job)
+              end
+            end
+          end
+
+          new_workload = {}
+          workload.each do |k,v|
+            next if k.done? || k.error? || k.aborted?
+            #new_workload[k] = v.reject{|d| d.done? || ((d.error? || d.aborted?) && ! d.recoverable_error?)}
+            new_workload[k] = v.reject{|d| d.done? || d.error? || d.aborted?}
+          end
+          workload = new_workload
+          sleep timer
+        end
+      rescue TryAgain
+        retry
+      end
+    end
+  end
+end

data/lib/scout/workflow/deployment.rb

@@ -0,0 +1 @@
+require_relative 'deployment/orchestrator'

data/lib/scout/workflow/step/dependencies.rb

@@ -1,7 +1,13 @@
 class Step
-  def rec_dependencies
-    rec_dependencies = dependencies.dup
-    dependencies.inject(rec_dependencies){|acc,d| acc.concat d.rec_dependencies }
+  def rec_dependencies(connected = false, seen = [])
+    direct_deps = []
+    dependencies.each do |dep|
+      next if seen.include? dep.path
+      next if connected && dep.done? && dep.updated?
+      direct_deps << dep
+    end
+    seen.concat direct_deps.collect{|d| d.path }
+    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, [])); acc }
   end
 
   def recursive_inputs
@@ -13,15 +19,21 @@ class Step
 
   def input_dependencies
     return [] unless inputs
-    inputs.select do |d|
-      Step === d
-    end
+    inputs.collect do |d|
+      if Step === d
+        d
+      elsif (Path === d) && (Step === d.pkgdir)
+        d.pkgdir
+      else
+        nil
+      end
+    end.compact.uniq
   end
 
   def prepare_dependencies
     inverse_dep = {}
-    dependencies.each{|dep| 
-      if dep.present? && ! dep.updated? 
+    dependencies.each{|dep|
+      if dep.present? && ! dep.updated?
         Log.debug "Clean outdated #{dep.path}"
         dep.clean
       end
@@ -29,12 +41,12 @@ class Step
       if dep.dependencies
         dep.dependencies.each do |d|
           inverse_dep[d] ||= []
-          inverse_dep[d] << dep 
+          inverse_dep[d] << dep
         end
       end
       input_dependencies.each do |d|
         inverse_dep[d] ||= []
-        inverse_dep[d] << dep 
+        inverse_dep[d] << dep
       end
     }
     inverse_dep.each do |dep,list|
@@ -43,7 +55,25 @@ class Step
   end
 
   def run_dependencies
-    dependencies.each{|dep| dep.run(true) unless dep.running? || dep.done? }
+    dependencies.each{|dep| 
+      next if dep.running? || dep.done?
+      compute_options = compute[dep.path] if compute
+      compute_options = [] if compute_options.nil?
+
+      stream = compute_options.include?(:stream)
+      stream = true unless ENV["SCOUT_EXPLICIT_STREAMING"] == 'true'
+      stream = false if compute_options.include?(:produce)
+
+      begin
+        dep.run(stream)
+      rescue ScoutException
+        if compute_options.include?(:canfail)
+          Log.medium "Allow failing of #{dep.path}"
+        else
+          raise $!
+        end
+      end
+    }
   end
 
   def abort_dependencies
@@ -52,9 +82,11 @@ class Step
 
   def self.wait_for_jobs(jobs)
     threads = []
-    jobs.each do |job| 
-      threads << job.join
+    jobs.each do |job|
+      threads << Thread.new{ job.join }
+    end
+    threads.each do |t|
+      t.join
     end
-    threads.each do |t| t.join end
   end
 end

data/lib/scout/workflow/step/file.rb

@@ -3,6 +3,7 @@ class Step
     @files_dir ||= begin
                      dir = @path + ".files"
                      @path.annotate(dir) if Path === @path
+                     dir.pkgdir = self
                      dir
                    end
   end
@@ -12,4 +13,8 @@ class Step
     Path.setup(dir) unless Path === dir
     dir[file]
   end
+
+  def bundle_files
+    [path, info_file, Dir.glob(File.join(files_dir,"**/*"))].flatten.select{|f| Open.exist?(f) }
+  end
 end

data/lib/scout/workflow/step/info.rb

@@ -43,7 +43,7 @@ class Step
     new_info.each do |key,value|
       if key == :status
         message = new_info[:messages]
-        if message.nil? && value == :done || value == :error || value == :aborted
+        if message.nil? && (value == :done || value == :error || value == :aborted)
           start = info[:start]
           eend = new_info[:end]
           if start && eend
@@ -54,6 +54,7 @@ class Step
         end
         report_status value, message 
       end
+
       if Exception === value
         begin
           Marshal.dump(value)
@@ -67,6 +68,7 @@ class Step
           value = new
         end
       end
+
       if info.include?(key)
         case info[key]
         when Array
@@ -114,17 +116,25 @@ class Step
   end
 
   def error?
-    status == :error
+    status == :error || status == 'error'
   end
 
   def aborted?
-    status == :aborted
+    status == :aborted || status == 'aborted'
   end
 
   def running?
     ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
   end
 
+  def overriden?
+    overriden_task || overriden_workflow || dependencies.select{|d| d.overriden? }.any?
+  end
+
+  def overriden_deps
+    rec_dependencies.select{|d| d.overriden? }
+  end
+
   def exception
     info[:exception]
   end

data/lib/scout/workflow/step/inputs.rb

@@ -0,0 +1,5 @@
+class Step
+  def save_inputs(inputs_dir)
+    self.task.save_inputs(inputs_dir, provided_inputs)
+  end
+end

data/lib/scout/workflow/step/load.rb

@@ -12,7 +12,7 @@ class Step
 
   def self.load(path)
     path = relocate(path) unless Open.exists?(path)
-    raise "Could not load #{path}" unless Open.exists?(path)
+    #raise "Could not load #{path}" unless Open.exists?(path)
     s = Step.new path
   end
 end

data/lib/scout/workflow/step/provenance.rb

@@ -91,6 +91,7 @@ class Step
     info[:task_name] = task
     path  = step.path
     status = info[:status] || :missing
+    status = status.to_sym if String === status
     status = :noinfo if status == :missing && Open.exist?(path)
     status = "remote" if Open.remote?(path) || Open.ssh?(path)
     name = info[:name] || File.basename(path)

data/lib/scout/workflow/step/status.rb

@@ -1,7 +1,11 @@
 class Step
   def abort(exception = nil)
-    while @result && streaming? && stream = self.stream
-      stream.abort(exception)
+    if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
+      Process.kill pid
+    else
+      while @result && streaming? && stream = self.stream
+        stream.abort(exception)
+      end
     end
   end
 
@@ -11,7 +15,7 @@ class Step
 
   def updated?
     return false if self.error? && self.recoverable_error?
-    return true unless ENV["SCOUT_UPDATE"]
+    return true if self.done? && ! ENV["SCOUT_UPDATE"]
     newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
 
@@ -24,16 +28,11 @@ class Step
     @info = nil
     @info_load_time = nil
     Open.rm path if Open.exist?(path)
+    Open.rm tmp_path if Open.exist?(tmp_path)
     Open.rm info_file if Open.exist?(info_file)
     Open.rm_rf files_dir if Open.exist?(files_dir)
   end
 
-  def present?
-    Open.exist?(path) &&
-      Open.exist?(info_file) &&
-      Open.exist?(files_dir)
-  end
-
 
   def recursive_clean
     dependencies.each do |dep|
@@ -42,4 +41,23 @@ class Step
     clean
   end
 
+  def canfail?
+    @compute && @compute.include?(:canfail)
+  end
+
+  def started?
+    return true if done?
+    return false unless Open.exist?(info_file)
+    pid = info[:pid]
+    return false unless pid
+    return Misc.pid_alive?(pid)
+  end
+
+  def waiting?
+    present? and not started?
+  end
+
+  def dirty?
+    done? && ! updated?
+  end
 end