scout-gear 10.4.0 → 10.7.0

data/lib/scout/work_queue.rb

@@ -6,18 +6,29 @@ require 'timeout'
 class WorkQueue
   attr_accessor :workers, :worker_proc, :callback
 
+  def new_worker
+    worker = Worker.new
+    worker.queue_id = queue_id
+    worker
+  end
+
   def initialize(workers = 0, &block)
     workers = workers.to_i if String === workers
     @input = WorkQueue::Socket.new
     @output = WorkQueue::Socket.new
-    @workers = workers.times.collect{ Worker.new }
+    @workers = workers.times.collect{ new_worker }
     @worker_proc = block
     @worker_mutex = Mutex.new
     @removed_workers = []
+    Log.medium "Starting queue #{queue_id} with workers: #{Log.fingerprint @workers.collect{|w| w.worker_short_id }} and sockets #{@input.socket_id} and #{@output.socket_id}"
+  end
+
+  def queue_id
+    [object_id, Process.pid] * "@"
   end
 
   def add_worker(&block)
-    worker = Worker.new
+    worker = new_worker
     @worker_mutex.synchronize do
       @workers.push(worker)
       if block_given?
@@ -41,9 +52,11 @@ class WorkQueue
     @worker_mutex.synchronize do
       worker = @workers.index{|w| w.pid == pid}
       if worker
-        Log.low "Removed worker #{pid}"
         @workers.delete_at(worker)
         @removed_workers << pid
+        Log.low "Removed worker #{pid} from #{queue_id}"
+      else
+        Log.medium "Worker #{pid} not from #{queue_id}"
       end
     end
   end
@@ -56,14 +69,14 @@ class WorkQueue
     @reader = Thread.new(Thread.current) do |parent|
       begin
         Thread.current.report_on_exception = false
-        Thread.current["name"] = "Output reader #{Process.pid}"
+        Thread.current["name"] = "Output reader #{queue_id}"
         @done_workers ||= []
         while true
           obj = @output.read
           if DoneProcessing === obj
 
             done = @worker_mutex.synchronize do
-              Log.low "Worker #{obj.pid} done"
+              Log.low "Worker #{obj.pid} from #{queue_id} done"
               @done_workers << obj.pid
               @closed && @done_workers.length == @removed_workers.length + @workers.length
             end
@@ -78,12 +91,12 @@ class WorkQueue
       rescue DoneProcessing
       rescue Aborted
       rescue WorkerException
-        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.worker_exception.message}"
+        Log.error "Exception in worker #{obj.pid} in queue #{queue_id}: #{obj.worker_exception.message}"
         self.abort
         @input.abort obj.worker_exception
         raise obj.worker_exception
       rescue
-        Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
+        Log.error "Exception processing output in queue #{queue_id}: #{$!.message}"
         self.abort
         raise $!
       end
@@ -95,25 +108,19 @@ class WorkQueue
 
     @waiter = Thread.new do
       Thread.current.report_on_exception = false
-      Thread.current["name"] = "Worker waiter #{Process.pid}"
+      Thread.current["name"] = "Worker waiter #{queue_id}"
       while true
         break if @worker_mutex.synchronize{ @workers.empty? }
-        begin
-          Timeout.timeout(1) do
-            begin
-              pid, status = Process.wait2
-              remove_worker(pid) if pid
-            rescue Exception
-              Log.exception $!
-            end
-          end
-        rescue Timeout::Error
-          pids = @worker_mutex.synchronize{ @workers.collect{|w| w.pid } }
-          pids.each do |p|
-            pid, status = Process.wait2 p, Process::WNOHANG
+        threads = @workers.collect do |w|
+          t = Thread.new do
+            Thread.current["name"] = "Worker waiter #{queue_id} worker #{w.pid}"
+            pid, status = Process.wait2 w.pid
             remove_worker(pid) if pid
           end
+          Thread.pass until t["name"]
+          t
         end
+        threads.each do |t| t.join end
       end
     end
 
@@ -131,16 +138,25 @@ class WorkQueue
   end
 
   def abort
-    Log.low "Aborting #{@workers.length} workers in queue #{Process.pid}"
+    @aborted = true
+    Log.low "Aborting #{@workers.length} workers in queue #{queue_id}"
     @worker_mutex.synchronize do
-      @workers.each{|w| w.abort }
+      @workers.each do |w| 
+        ScoutSemaphore.post_semaphore(@output.write_sem)
+        ScoutSemaphore.post_semaphore(@input.read_sem)
+        w.abort 
+      end
     end
   end
 
   def close
+    return if @closed || @aborted
     @closed = true
     @worker_mutex.synchronize{ @workers.length }.times do
-      @input.write DoneProcessing.new() unless @input.closed_write?
+      begin
+        @input.write DoneProcessing.new() unless @input.closed_write?
+      rescue IOError
+      end
     end
   end
 
@@ -151,6 +167,7 @@ class WorkQueue
   end
 
   def join(clean = true)
+    close
     begin
       @waiter.join if @waiter
       @reader.join if @reader

data/lib/scout/workflow/definition.rb

@@ -1,8 +1,8 @@
-require 'scout/meta_extension'
+require 'scout/annotation'
 
 module Workflow
-  extend MetaExtension
-  extension_attr :name, :tasks, :helpers
+  extend Annotation
+  annotation :name, :tasks, :helpers
 
   class << self
     attr_accessor :directory
@@ -21,12 +21,14 @@ module Workflow
     @helpers ||= {}
   end
 
-  def helper(name, *args, &block)
+  def helper(name, *args, **kwargs, &block)
     if block_given?
       helpers[name] = block
     else
       raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
-      helpers[name].call(*args)
+      o = Object.new
+      o.extend step_module
+      o.send(name, *args, **kwargs)
     end
   end
 
@@ -143,10 +145,11 @@ module Workflow
   def task_alias(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
-    task_proc = workflow.tasks[oname]
-    raise "Task #{oname} not found" if task_proc.nil?
-    returns task_proc.returns if @returns.nil?
-    type = task_proc.type 
+    task_proc = workflow.tasks[oname] if workflow.tasks
+    if task_proc
+      returns task_proc.returns if @returns.nil?
+      type = task_proc.type 
+    end
     task name => type do
       raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
@@ -154,18 +157,18 @@ module Workflow
       dep.join
       raise dep.get_exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
-      set_info :result_type, dep.info[:result_type]
+      set_info :type, dep.info[:type]
       forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
         remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
 
         self.archive_deps
-        self.copy_files_dir
+        self.copy_linked_files_dir
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
 
-        if dep.overriden || ! Workflow.job_path?(dep.path)
+        if dep.overriden? 
           Open.link dep.path, self.tmp_path
         else
           Open.ln_h dep.path, self.tmp_path

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -34,7 +34,7 @@ module Workflow
     end
 
     def self.job_rules(rules, job)
-      workflow = job.workflow.to_s
+      workflow = job.workflow.name
       task_name = job.task_name.to_s
       defaults = rules["defaults"] || {}
 
@@ -168,7 +168,7 @@ module Workflow
 
         dep_path = dep.path
         parents = all_jobs.select do |parent| 
-          paths = parent.info[:dependencies].nil? ? parent.dependencies.collect{|d| d.path } : parent.info[:dependencies].collect{|d| d.last }
+          paths = parent.info[:dependencies].nil? ? parent.dependencies.collect{|d| d.path } : parent.info[:dependencies].collect{|d| Array === d ? d.last : d }
           paths.include? dep_path
         end
 
@@ -177,7 +177,7 @@ module Workflow
         parents.each do |parent|
           Log.high "Erasing #{dep.path} from #{parent.path}"
           parent.archive_deps
-          parent.copy_files_dir
+          parent.copy_linked_files_dir
           parent.dependencies = parent.dependencies - [dep]
         end
         dep.clean
@@ -246,9 +246,27 @@ module Workflow
           workload = new_workload
           sleep timer
         end
+        all_jobs.each{|s| s.join }
       rescue TryAgain
         retry
       end
     end
   end
+
+  def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors)
+    jobs = [jobs] unless Array === jobs
+    produce_list = []
+    jobs.each do |job|
+      next if job.done? || job.running?
+      job.rec_dependencies.each do |job|
+        produce_list << job if tasks.include?(job.task_name) ||
+          tasks.include?(job.task_name.to_s) ||
+          tasks.include?(job.full_task_name)
+      end
+    end
+
+    orchestrator = Orchestrator.new 0.1, cpus: produce_cpus.to_i
+    orchestrator.process({}, produce_list)
+    produce_list
+  end
 end

data/lib/scout/workflow/deployment/trace.rb

@@ -0,0 +1,205 @@
+require 'scout/tsv'
+
+module Workflow
+  def self.trace_job_times(jobs, fix_gap = false, report_keys = nil)
+    data = TSV.setup({}, "Job~Code,Workflow,Task,Start,End#:type=:list")
+    min_start = nil
+    max_done = nil
+    jobs.each do |job|
+      next unless job.info[:end]
+      started = job.info[:start]
+      ddone = job.info[:end]
+
+      started = Time.parse started if String === started
+      ddone = Time.parse ddone if String === ddone
+
+      code = [job.workflow.name, job.task_name].compact.collect{|s| s.to_s} * " · "
+      code = job.name + " - " + code
+
+      data[job.path] = [code, job.workflow.name, job.task_name, started, ddone]
+      if min_start.nil?
+        min_start = started
+      else
+        min_start = started if started < min_start
+      end
+
+      if max_done.nil?
+        max_done = ddone
+      else
+        max_done = ddone if ddone > max_done
+      end
+    end
+
+    data.add_field "Start.second" do |k,value|
+      value["Start"] - min_start
+    end
+
+    data.add_field "End.second" do |k,value|
+      value["End"] - min_start
+    end
+
+    if fix_gap
+      ranges = []
+      data.through do |k,values|
+        start, eend = values.values_at "Start.second", "End.second"
+
+        ranges << (start..eend)
+      end
+
+      gaps = {}
+      last = nil
+      Misc.collapse_ranges(ranges).each do |range|
+        start = range.begin
+        eend = range.end
+        if last
+          gaps[last] = start - last
+        end
+        last = eend
+      end
+
+      data.process "End.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+
+      data.process "Start.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+
+      total_gaps = Misc.sum(gaps.collect{|k,v| v})
+      Log.info "Total gaps: #{total_gaps} seconds"
+    end
+
+    if report_keys && report_keys.any?
+      job_keys = {}
+      jobs.each do |job|
+        job_info = IndiferentHash.setup(job.info)
+        report_keys.each do |key|
+          job_keys[job.path] ||= {}
+          job_keys[job.path][key] = job_info[key]
+        end
+      end
+      report_keys.each do |key|
+        data.add_field Misc.humanize(key) do |p,values|
+          job_keys[p][key]
+        end
+      end
+    end
+
+    start = data.column("Start.second").values.flatten.collect{|v| v.to_f}.min
+    eend = data.column("End.second").values.flatten.collect{|v| v.to_f}.max
+    total = eend - start unless eend.nil? || start.nil?
+    Log.info "Total time elapsed: #{total} seconds" if total
+
+    if report_keys && report_keys.any?
+      job_keys = {}
+      report_keys.each do |key|
+        jobs.each do |job|
+          job_keys[job.path] ||= {}
+          job_keys[job.path][key] = job.info[key]
+        end
+      end
+      report_keys.each do |key|
+        data.add_field Misc.humanize(key) do |p,values|
+          job_keys[p][key]
+        end
+      end
+    end
+
+    data
+  end
+
+  def self.trace_job_summary(jobs, report_keys = [])
+    tasks_info = {}
+
+    report_keys = report_keys.collect{|k| k.to_s}
+
+    jobs.each do |dep|
+      next unless dep.info[:end]
+      task = [dep.workflow.name, dep.task_name].compact.collect{|s| s.to_s} * "#"
+      info = tasks_info[task] ||= IndiferentHash.setup({})
+      dep_info = IndiferentHash.setup(dep.info)
+
+      ddone = dep_info[:end]
+      started = dep_info[:start]
+
+      started = Time.parse started if String === started
+      ddone = Time.parse ddone if String === ddone
+
+      time = ddone - started
+      info[:time] ||= []
+      info[:time] << time
+
+      report_keys.each do |key|
+        info[key] = dep_info[key] 
+      end
+
+      dep.info[:config_keys].each do |kinfo| 
+        key, value, tokens = kinfo
+
+        info[key.to_s] = value if report_keys.include? key.to_s
+      end if dep.info[:config_keys]
+    end
+
+    summary = TSV.setup({}, "Task~Calls,Avg. Time,Total Time#:type=:list")
+
+    tasks_info.each do |task, info|
+      time_lists = info[:time]
+      avg_time = Misc.mean(time_lists).to_i
+      total_time = Misc.sum(time_lists).to_i
+      calls = time_lists.length
+      summary[task] = [calls, avg_time, total_time]
+    end
+
+    report_keys.each do |key|
+      summary.add_field Misc.humanize(key) do |task|
+        tasks_info[task][key]
+      end
+    end if Array === report_keys && report_keys.any?
+
+    summary
+  end
+
+  def self.trace(seed_jobs, options = {})
+    jobs = []
+    seed_jobs.each do |step|
+      jobs += step.rec_dependencies + [step]
+      step.info[:archived_info].each do |path,ainfo|
+        next unless Hash === ainfo
+        archived_step = Step.new path
+
+        archived_step.define_singleton_method :info do
+          ainfo
+        end
+
+        jobs << archived_step
+      end if step.info[:archived_info]
+
+    end
+
+    jobs = jobs.uniq.sort_by{|job| [job, job.info]; t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
+
+    report_keys = options[:report_keys] || ""
+    report_keys = report_keys.split(/,\s*/) if String === report_keys
+
+    data = trace_job_times(jobs, options[:fix_gap], report_keys)
+
+    summary = trace_job_summary(jobs, report_keys)
+
+
+    raise "No jobs to process" if data.size == 0
+
+    size, width, height = options.values_at :size, :width, :height
+
+    size = 800 if size.nil?
+    width = size.to_i * 2 if width.nil?
+    height = size  if height.nil?
+
+    if options[:plot_data]
+      data
+    else
+      summary
+    end
+  end
+end

data/lib/scout/workflow/deployment.rb

@@ -1 +1,2 @@
 require_relative 'deployment/orchestrator'
+require_relative 'deployment/trace'

data/lib/scout/workflow/documentation.rb

@@ -31,7 +31,7 @@ module Workflow
     title = doc_parse_first_line doc
     description, task_info = doc_parse_up_to doc, /^# Tasks/i
     task_description, tasks = doc_parse_up_to task_info, /^##/, true
-    tasks = doc_parse_chunks tasks, /## (.*)/ 
+    tasks = doc_parse_chunks tasks, /^## (.*)/ 
     {:title => title.strip, :description => description.strip, :task_description => task_description.strip, :tasks => tasks}
   end
 

data/lib/scout/workflow/step/archive.rb

@@ -0,0 +1,42 @@
+class Step
+  def archived_info
+    return {} unless Open.exists?(info_file)
+    info[:archived_info] || {}
+  end
+
+  def archived_inputs
+    return [] unless info[:archived_dependencies]
+    archived_info = self.archived_info
+
+    all_inputs = NamedArray.setup([],[])
+    deps = info[:archived_dependencies].dup
+    seen = []
+    while path = deps.pop
+      dep_info = archived_info[path]
+      if Hash === dep_info
+        dep_inputs = dep_info[:inputs]
+        NamedArray.setup(dep_inputs, dep_info[:input_names])
+        all_inputs.concat(dep_inputs)
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
+      seen << path
+    end
+
+    all_inputs
+  end
+
+  def archive_deps(jobs = nil)
+    jobs = dependencies if jobs.nil?
+
+    archived_info = jobs.inject({}) do |acc,dep|
+      next acc unless Open.exists?(dep.info_file)
+      acc[dep.path] = dep.info
+      acc.merge!(dep.archived_info)
+      acc
+    end
+
+    self.set_info :archived_info, archived_info
+    self.set_info :archived_dependencies, info[:dependencies]
+  end
+end

data/lib/scout/workflow/step/children.rb

@@ -0,0 +1,51 @@
+class Step
+  def child(&block)
+    child_pid = Process.fork &block
+    children_pids = info[:children_pids]
+    if children_pids.nil?
+      children_pids = [child_pid]
+    else
+      children_pids << child_pid
+    end
+    set_info :children_pids, children_pids
+    child_pid
+  end
+
+  def cmd(*args)
+    all_args = *args
+
+    all_args << {} unless Hash === all_args.last
+
+    level = all_args.last[:log] || 0
+    level = 0 if TrueClass === level
+    level = 10 if FalseClass === level
+    level = level.to_i
+
+    all_args.last[:log] = true
+    all_args.last[:pipe] = true
+
+    io = CMD.cmd(*all_args)
+    child_pid = io.pids.first
+
+    children_pids = info[:children_pids]
+    if children_pids.nil?
+      children_pids = [child_pid]
+    else
+      children_pids << child_pid
+    end
+    set_info :children_pids, children_pids
+
+    while c = io.getc
+      STDERR << c if Log.severity <= level
+      if c == "\n"
+        Log.logn "STDOUT [#{child_pid}]: ", level
+      end
+    end 
+
+    io.join
+
+    nil
+  end
+
+end
+

data/lib/scout/workflow/step/config.rb

@@ -7,7 +7,7 @@ class Step
 
     new_tokens = []
     if workflow
-      workflow_name = workflow.to_s
+      workflow_name = workflow.name
       new_tokens << ("workflow:" << workflow_name)
       new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
     end

data/lib/scout/workflow/step/dependencies.rb

@@ -33,37 +33,51 @@ class Step
 
   def prepare_dependencies
     inverse_dep = {}
-    dependencies.each{|dep|
+
+    dependencies.each do |dep|
       if dep.present? && ! dep.updated?
-        Log.debug "Clean outdated #{dep.path}"
+        Log.medium "Clean outdated #{dep.path}"
         dep.clean
       end
+
       next if dep.done?
+
       if dep.dependencies
         dep.dependencies.each do |d|
           inverse_dep[d] ||= []
           inverse_dep[d] << dep
         end
       end
+
       input_dependencies.each do |d|
         inverse_dep[d] ||= []
         inverse_dep[d] << dep
       end
-    }
+    end if dependencies
+
     inverse_dep.each do |dep,list|
       dep.tee_copies = list.length
     end
   end
 
+  def all_dependencies
+    @all_dependencies ||= begin
+                            all_dependencies = []
+                            all_dependencies += dependencies if dependencies
+                            all_dependencies += input_dependencies if input_dependencies
+                            all_dependencies
+                          end
+  end
+
   def run_dependencies
-    dependencies.each{|dep| 
+    all_dependencies.each do |dep| 
       next if dep.running? || dep.done?
       compute_options = compute[dep.path] if compute
       compute_options = [] if compute_options.nil?
 
       stream = compute_options.include?(:stream)
       stream = true unless ENV["SCOUT_EXPLICIT_STREAMING"] == 'true'
-      stream = false if compute_options.include?(:produce)
+      stream = :no_load if compute_options.include?(:produce)
 
       begin
         dep.run(stream)
@@ -74,17 +88,20 @@ class Step
           raise $!
         end
       end
-    }
+    end
   end
 
   def abort_dependencies
-    dependencies.each{|dep| dep.abort if dep.running? }
+    all_dependencies.each{|dep| dep.abort if dep.running? } 
   end
 
   def self.wait_for_jobs(jobs)
     threads = []
     jobs.each do |job|
-      threads << Thread.new{ job.join }
+      threads << Thread.new do
+        Thread.current.report_on_exception = false
+        job.join
+      end
     end
     threads.each do |t|
       t.join

data/lib/scout/workflow/step/file.rb

@@ -19,7 +19,26 @@ class Step
     dir[file]
   end
 
+  def files
+    Dir.glob(File.join(files_dir, '**', '*')).reject{|path| File.directory? path }.collect do |path| 
+      Misc.path_relative_to(files_dir, path) 
+    end
+  end
+
   def bundle_files
     [path, info_file, Dir.glob(File.join(files_dir,"**/*"))].flatten.select{|f| Open.exist?(f) }
   end
+
+  def copy_linked_files_dir
+    if File.symlink?(self.files_dir)
+      begin
+        realpath = Open.realpath(self.files_dir)
+        Open.rm self.files_dir
+        Open.cp realpath, self.files_dir
+      rescue
+        Log.warn "Copy files_dir for #{self.workflow_short_path} failed: " + $!.message
+      end
+    end
+  end
+
 end