scout-gear 10.4.0 → 10.6.1

data/lib/scout/workflow/definition.rb

@@ -1,8 +1,8 @@
-require 'scout/meta_extension'
+require 'scout/annotation'
 
 module Workflow
-  extend MetaExtension
-  extension_attr :name, :tasks, :helpers
+  extend Annotation
+  annotation :name, :tasks, :helpers
 
   class << self
     attr_accessor :directory
@@ -143,10 +143,11 @@ module Workflow
   def task_alias(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
-    task_proc = workflow.tasks[oname]
-    raise "Task #{oname} not found" if task_proc.nil?
-    returns task_proc.returns if @returns.nil?
-    type = task_proc.type 
+    task_proc = workflow.tasks[oname] if workflow.tasks
+    if task_proc
+      returns task_proc.returns if @returns.nil?
+      type = task_proc.type 
+    end
     task name => type do
       raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
@@ -154,18 +155,18 @@ module Workflow
       dep.join
       raise dep.get_exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
-      set_info :result_type, dep.info[:result_type]
+      set_info :type, dep.info[:type]
       forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
         remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
 
         self.archive_deps
-        self.copy_files_dir
+        self.copy_linked_files_dir
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
 
-        if dep.overriden || ! Workflow.job_path?(dep.path)
+        if dep.overriden? 
           Open.link dep.path, self.tmp_path
         else
           Open.ln_h dep.path, self.tmp_path

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -34,7 +34,7 @@ module Workflow
     end
 
     def self.job_rules(rules, job)
-      workflow = job.workflow.to_s
+      workflow = job.workflow.name
       task_name = job.task_name.to_s
       defaults = rules["defaults"] || {}
 
@@ -168,7 +168,7 @@ module Workflow
 
         dep_path = dep.path
         parents = all_jobs.select do |parent| 
-          paths = parent.info[:dependencies].nil? ? parent.dependencies.collect{|d| d.path } : parent.info[:dependencies].collect{|d| d.last }
+          paths = parent.info[:dependencies].nil? ? parent.dependencies.collect{|d| d.path } : parent.info[:dependencies].collect{|d| Array === d ? d.last : d }
           paths.include? dep_path
         end
 
@@ -177,7 +177,7 @@ module Workflow
         parents.each do |parent|
           Log.high "Erasing #{dep.path} from #{parent.path}"
           parent.archive_deps
-          parent.copy_files_dir
+          parent.copy_linked_files_dir
           parent.dependencies = parent.dependencies - [dep]
         end
         dep.clean
@@ -251,4 +251,21 @@ module Workflow
       end
     end
   end
+
+  def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors)
+    jobs = [jobs] unless Array === jobs
+    produce_list = []
+    jobs.each do |job|
+      next if job.done? || job.running?
+      job.rec_dependencies.each do |job|
+        produce_list << job if tasks.include?(job.task_name) ||
+          tasks.include?(job.task_name.to_s) ||
+          tasks.include?(job.full_task_name)
+      end
+    end
+
+    orchestrator = Orchestrator.new 0.1, cpus: produce_cpus.to_i
+    orchestrator.process({}, produce_list)
+    produce_list
+  end
 end

data/lib/scout/workflow/deployment/trace.rb

@@ -0,0 +1,205 @@
+require 'scout/tsv'
+
+module Workflow
+  def self.trace_job_times(jobs, fix_gap = false, report_keys = nil)
+    data = TSV.setup({}, "Job~Code,Workflow,Task,Start,End#:type=:list")
+    min_start = nil
+    max_done = nil
+    jobs.each do |job|
+      next unless job.info[:end]
+      started = job.info[:start]
+      ddone = job.info[:end]
+
+      started = Time.parse started if String === started
+      ddone = Time.parse ddone if String === ddone
+
+      code = [job.workflow.name, job.task_name].compact.collect{|s| s.to_s} * " · "
+      code = job.name + " - " + code
+
+      data[job.path] = [code, job.workflow.name, job.task_name, started, ddone]
+      if min_start.nil?
+        min_start = started
+      else
+        min_start = started if started < min_start
+      end
+
+      if max_done.nil?
+        max_done = ddone
+      else
+        max_done = ddone if ddone > max_done
+      end
+    end
+
+    data.add_field "Start.second" do |k,value|
+      value["Start"] - min_start
+    end
+
+    data.add_field "End.second" do |k,value|
+      value["End"] - min_start
+    end
+
+    if fix_gap
+      ranges = []
+      data.through do |k,values|
+        start, eend = values.values_at "Start.second", "End.second"
+
+        ranges << (start..eend)
+      end
+
+      gaps = {}
+      last = nil
+      Misc.collapse_ranges(ranges).each do |range|
+        start = range.begin
+        eend = range.end
+        if last
+          gaps[last] = start - last
+        end
+        last = eend
+      end
+
+      data.process "End.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+
+      data.process "Start.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+
+      total_gaps = Misc.sum(gaps.collect{|k,v| v})
+      Log.info "Total gaps: #{total_gaps} seconds"
+    end
+
+    if report_keys && report_keys.any?
+      job_keys = {}
+      jobs.each do |job|
+        job_info = IndiferentHash.setup(job.info)
+        report_keys.each do |key|
+          job_keys[job.path] ||= {}
+          job_keys[job.path][key] = job_info[key]
+        end
+      end
+      report_keys.each do |key|
+        data.add_field Misc.humanize(key) do |p,values|
+          job_keys[p][key]
+        end
+      end
+    end
+
+    start = data.column("Start.second").values.flatten.collect{|v| v.to_f}.min
+    eend = data.column("End.second").values.flatten.collect{|v| v.to_f}.max
+    total = eend - start unless eend.nil? || start.nil?
+    Log.info "Total time elapsed: #{total} seconds" if total
+
+    if report_keys && report_keys.any?
+      job_keys = {}
+      report_keys.each do |key|
+        jobs.each do |job|
+          job_keys[job.path] ||= {}
+          job_keys[job.path][key] = job.info[key]
+        end
+      end
+      report_keys.each do |key|
+        data.add_field Misc.humanize(key) do |p,values|
+          job_keys[p][key]
+        end
+      end
+    end
+
+    data
+  end
+
+  def self.trace_job_summary(jobs, report_keys = [])
+    tasks_info = {}
+
+    report_keys = report_keys.collect{|k| k.to_s}
+
+    jobs.each do |dep|
+      next unless dep.info[:end]
+      task = [dep.workflow.name, dep.task_name].compact.collect{|s| s.to_s} * "#"
+      info = tasks_info[task] ||= IndiferentHash.setup({})
+      dep_info = IndiferentHash.setup(dep.info)
+
+      ddone = dep_info[:end]
+      started = dep_info[:start]
+
+      started = Time.parse started if String === started
+      ddone = Time.parse ddone if String === ddone
+
+      time = ddone - started
+      info[:time] ||= []
+      info[:time] << time
+
+      report_keys.each do |key|
+        info[key] = dep_info[key] 
+      end
+
+      dep.info[:config_keys].each do |kinfo| 
+        key, value, tokens = kinfo
+
+        info[key.to_s] = value if report_keys.include? key.to_s
+      end if dep.info[:config_keys]
+    end
+
+    summary = TSV.setup({}, "Task~Calls,Avg. Time,Total Time#:type=:list")
+
+    tasks_info.each do |task, info|
+      time_lists = info[:time]
+      avg_time = Misc.mean(time_lists).to_i
+      total_time = Misc.sum(time_lists).to_i
+      calls = time_lists.length
+      summary[task] = [calls, avg_time, total_time]
+    end
+
+    report_keys.each do |key|
+      summary.add_field Misc.humanize(key) do |task|
+        tasks_info[task][key]
+      end
+    end if Array === report_keys && report_keys.any?
+
+    summary
+  end
+
+  def self.trace(seed_jobs, options = {})
+    jobs = []
+    seed_jobs.each do |step|
+      jobs += step.rec_dependencies + [step]
+      step.info[:archived_info].each do |path,ainfo|
+        next unless Hash === ainfo
+        archived_step = Step.new path
+
+        archived_step.define_singleton_method :info do
+          ainfo
+        end
+
+        jobs << archived_step
+      end if step.info[:archived_info]
+
+    end
+
+    jobs = jobs.uniq.sort_by{|job| [job, job.info]; t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
+
+    report_keys = options[:report_keys] || ""
+    report_keys = report_keys.split(/,\s*/) if String === report_keys
+
+    data = trace_job_times(jobs, options[:fix_gap], report_keys)
+
+    summary = trace_job_summary(jobs, report_keys)
+
+
+    raise "No jobs to process" if data.size == 0
+
+    size, width, height = options.values_at :size, :width, :height
+
+    size = 800 if size.nil?
+    width = size.to_i * 2 if width.nil?
+    height = size  if height.nil?
+
+    if options[:plot_data]
+      data
+    else
+      summary
+    end
+  end
+end

data/lib/scout/workflow/deployment.rb

@@ -1 +1,2 @@
 require_relative 'deployment/orchestrator'
+require_relative 'deployment/trace'

data/lib/scout/workflow/documentation.rb

@@ -31,7 +31,7 @@ module Workflow
     title = doc_parse_first_line doc
     description, task_info = doc_parse_up_to doc, /^# Tasks/i
     task_description, tasks = doc_parse_up_to task_info, /^##/, true
-    tasks = doc_parse_chunks tasks, /## (.*)/ 
+    tasks = doc_parse_chunks tasks, /^## (.*)/ 
     {:title => title.strip, :description => description.strip, :task_description => task_description.strip, :tasks => tasks}
   end
 

data/lib/scout/workflow/step/archive.rb

@@ -0,0 +1,42 @@
+class Step
+  def archived_info
+    return {} unless Open.exists?(info_file)
+    info[:archived_info] || {}
+  end
+
+  def archived_inputs
+    return [] unless info[:archived_dependencies]
+    archived_info = self.archived_info
+
+    all_inputs = NamedArray.setup([],[])
+    deps = info[:archived_dependencies].dup
+    seen = []
+    while path = deps.pop
+      dep_info = archived_info[path]
+      if Hash === dep_info
+        dep_inputs = dep_info[:inputs]
+        NamedArray.setup(dep_inputs, dep_info[:input_names])
+        all_inputs.concat(dep_inputs)
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
+      seen << path
+    end
+
+    all_inputs
+  end
+
+  def archive_deps(jobs = nil)
+    jobs = dependencies if jobs.nil?
+
+    archived_info = jobs.inject({}) do |acc,dep|
+      next acc unless Open.exists?(dep.info_file)
+      acc[dep.path] = dep.info
+      acc.merge!(dep.archived_info)
+      acc
+    end
+
+    self.set_info :archived_info, archived_info
+    self.set_info :archived_dependencies, info[:dependencies]
+  end
+end

data/lib/scout/workflow/step/children.rb

@@ -0,0 +1,51 @@
+class Step
+  def child(&block)
+    child_pid = Process.fork &block
+    children_pids = info[:children_pids]
+    if children_pids.nil?
+      children_pids = [child_pid]
+    else
+      children_pids << child_pid
+    end
+    set_info :children_pids, children_pids
+    child_pid
+  end
+
+  def cmd(*args)
+    all_args = *args
+
+    all_args << {} unless Hash === all_args.last
+
+    level = all_args.last[:log] || 0
+    level = 0 if TrueClass === level
+    level = 10 if FalseClass === level
+    level = level.to_i
+
+    all_args.last[:log] = true
+    all_args.last[:pipe] = true
+
+    io = CMD.cmd(*all_args)
+    child_pid = io.pids.first
+
+    children_pids = info[:children_pids]
+    if children_pids.nil?
+      children_pids = [child_pid]
+    else
+      children_pids << child_pid
+    end
+    set_info :children_pids, children_pids
+
+    while c = io.getc
+      STDERR << c if Log.severity <= level
+      if c == "\n"
+        Log.logn "STDOUT [#{child_pid}]: ", level
+      end
+    end 
+
+    io.join
+
+    nil
+  end
+
+end
+

data/lib/scout/workflow/step/config.rb

@@ -7,7 +7,7 @@ class Step
 
     new_tokens = []
     if workflow
-      workflow_name = workflow.to_s
+      workflow_name = workflow.name
       new_tokens << ("workflow:" << workflow_name)
       new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
     end

data/lib/scout/workflow/step/dependencies.rb

@@ -33,37 +33,51 @@ class Step
 
   def prepare_dependencies
     inverse_dep = {}
-    dependencies.each{|dep|
+
+    dependencies.each do |dep|
       if dep.present? && ! dep.updated?
         Log.debug "Clean outdated #{dep.path}"
         dep.clean
       end
+
       next if dep.done?
+
       if dep.dependencies
         dep.dependencies.each do |d|
           inverse_dep[d] ||= []
           inverse_dep[d] << dep
         end
       end
+
       input_dependencies.each do |d|
         inverse_dep[d] ||= []
         inverse_dep[d] << dep
       end
-    }
+    end if dependencies
+
     inverse_dep.each do |dep,list|
       dep.tee_copies = list.length
     end
   end
 
+  def all_dependencies
+    @all_dependencies ||= begin
+                            all_dependencies = []
+                            all_dependencies += dependencies if dependencies
+                            all_dependencies += input_dependencies if input_dependencies
+                            all_dependencies
+                          end
+  end
+
   def run_dependencies
-    dependencies.each{|dep| 
+    all_dependencies.each do |dep| 
       next if dep.running? || dep.done?
       compute_options = compute[dep.path] if compute
       compute_options = [] if compute_options.nil?
 
       stream = compute_options.include?(:stream)
       stream = true unless ENV["SCOUT_EXPLICIT_STREAMING"] == 'true'
-      stream = false if compute_options.include?(:produce)
+      stream = :no_load if compute_options.include?(:produce)
 
       begin
         dep.run(stream)
@@ -74,17 +88,20 @@ class Step
           raise $!
         end
       end
-    }
+    end
   end
 
   def abort_dependencies
-    dependencies.each{|dep| dep.abort if dep.running? }
+    all_dependencies.each{|dep| dep.abort if dep.running? } 
   end
 
   def self.wait_for_jobs(jobs)
     threads = []
     jobs.each do |job|
-      threads << Thread.new{ job.join }
+      threads << Thread.new do
+        Thread.current.report_on_exception = false
+        job.join
+      end
     end
     threads.each do |t|
       t.join

data/lib/scout/workflow/step/file.rb

@@ -19,7 +19,26 @@ class Step
     dir[file]
   end
 
+  def files
+    Dir.glob(File.join(files_dir, '**', '*')).reject{|path| File.directory? path }.collect do |path| 
+      Misc.path_relative_to(files_dir, path) 
+    end
+  end
+
   def bundle_files
     [path, info_file, Dir.glob(File.join(files_dir,"**/*"))].flatten.select{|f| Open.exist?(f) }
   end
+
+  def copy_linked_files_dir
+    if File.symlink?(self.files_dir)
+      begin
+        realpath = Open.realpath(self.files_dir)
+        Open.rm self.files_dir
+        Open.cp realpath, self.files_dir
+      rescue
+        Log.warn "Copy files_dir for #{self.workflow_short_path} failed: " + $!.message
+      end
+    end
+  end
+
 end

data/lib/scout/workflow/step/info.rb

@@ -19,15 +19,17 @@ class Step
     @info_load_time = Time.now
   end
 
-
-
   def save_info(info = nil)
     Persist.save(info, info_file, SERIALIZER)
     @info_load_time = Time.now
   end
 
-  def clear_info
-    save_info(@info = {})
+  def reset_info(info = {})
+    save_info(@info = info)
+  end
+
+  def init_info
+    log :waiting unless info_file.nil? || Open.exists?(info_file)
   end
 
   def info
@@ -47,20 +49,36 @@ class Step
   def merge_info(new_info)
     info = self.info
     new_info.each do |key,value|
+      value = Annotation.purge(value)
       if key == :status
-        message = new_info[:messages]
+        message = new_info[:message]
         if message.nil? && (value == :done || value == :error || value == :aborted)
+          issued = info[:issued]
           start = info[:start]
           eend = new_info[:end]
           if start && eend
             time = eend - start
-            time_str = Misc.format_seconds_short(time)
+            total_time = eend - issued
+            if total_time - time > 1
+              time_str = "#{Misc.format_seconds_short(time)} (#{Misc.format_seconds_short(total_time)})"
+            else
+              time_str = Misc.format_seconds_short(time)
+            end
+            info[:time_elapsed] = time
+            info[:total_time_elapsed] = total_time
             message = Log.color(:time, time_str)
           end
         end
         report_status value, message 
       end
 
+      if key == :message
+        messages = info[:messages] || []
+        messages << value
+        info[:messages] = messages
+        next
+      end
+
       if Exception === value
         begin
           Marshal.dump(value)
@@ -111,12 +129,16 @@ class Step
     end
 
     if message
-      merge_info :status => status, :messages => message
+      merge_info :status => status, :message => message
     else
       merge_info :status => status
     end
   end
 
+  def messages
+    info[:messages]
+  end
+
   def status
     info[:status].tap{|s| s.nil? ? s : s.to_sym }
   end
@@ -134,7 +156,8 @@ class Step
   end
 
   def overriden?
-    overriden_task || overriden_workflow || dependencies.select{|d| d.overriden? }.any?
+    @overriden = overriden_task || overriden_workflow || overriden_deps.any? if @overriden.nil?
+    @overriden
   end
 
   def overriden_deps
@@ -145,10 +168,15 @@ class Step
     info[:exception]
   end
 
-  def marshal_dump
+  # Marshal Step
+  def _dump(level)
     @path
   end
 
+  def self._load(path)
+    Step.new path
+  end
+
   def marshal_load(path)
     Step.new path
   end

data/lib/scout/workflow/step/progress.rb

@@ -1,5 +1,5 @@
 class Step
-  def progress_bar(msg = "Progress", options = nil)
+  def progress_bar(msg = "Progress", options = nil, &block)
     if Hash === msg and options.nil?
       options = msg
       msg = nil
@@ -8,7 +8,16 @@ class Step
 
     max = options[:max]
     Open.mkdir files_dir
-    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
+    bar = Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
+
+    if block_given?
+      bar.init
+      res = yield bar
+      bar.remove
+      res
+    else
+      bar
+    end
   end
 end
 

data/lib/scout/workflow/step/status.rb

@@ -1,11 +1,18 @@
 class Step
   def abort(exception = nil)
     if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
-      Process.kill pid
+      Log.debug "Kill process #{pid} to abort step #{Log.fingerprint self}"
+      begin
+        s = Misc.abort_child pid, true
+        Log.medium "Aborted pid #{path} #{s}"
+      rescue 
+        Log.debug("Aborted job #{pid} was not killed: #{$!.message}")
+      end
     else
       while @result && streaming? && stream = self.stream
         stream.abort(exception)
       end
+      @take_stream.abort(exception) if streaming?
     end
   end