scout-gear 6.0.0 → 7.2.0

data/lib/scout/work_queue.rb

@@ -35,52 +35,94 @@ class WorkQueue
   end
 
   def remove_worker(pid)
-    worker = @worker_mutex.synchronize do
-      Log.debug "Remove #{pid}"
-      @removed_workers.concat(@workers.delete_if{|w| w.pid == pid })
+    @worker_mutex.synchronize do
+      worker = @workers.index{|w| w.pid == pid}
+      if worker
+        Log.debug "Removed worker #{pid}"
+        @workers.delete_at(worker)
+        @removed_workers << pid
+      end
     end
   end
 
   def process(&callback)
-    @workers.each do |w| 
-      w.process @input, @output, &@worker_proc
-    end
-    @reader = Thread.new do
+    @reader = Thread.new do |parent|
       begin
+        Thread.current.report_on_exception = false
+        Thread.current["name"] = "Output reader #{Process.pid}"
+        @done_workers ||= []
         while true
           obj = @output.read
           if DoneProcessing === obj
-            remove_worker obj.pid if obj.pid
+
+            done = @worker_mutex.synchronize do
+              Log.low "Worker #{obj.pid} done"
+              @done_workers << obj.pid
+              @closed && @done_workers.length == @removed_workers.length + @workers.length
+            end
+
+            break if done
+          elsif Exception === obj
+            raise obj
           else
             callback.call obj if callback
           end
         end
+      rescue DoneProcessing
       rescue Aborted
+      rescue WorkerException
+        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
+        self.abort
+        raise obj.worker_exception
+      rescue
+        Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
+        self.abort
+        raise $!
+      end
+    end
+
+    @workers.each do |w| 
+      w.process @input, @output, &@worker_proc
+    end
+
+    Thread.pass until @reader["name"]
+
+    @waiter = Thread.new do
+      begin
+        Thread.current.report_on_exception = false
+        Thread.current["name"] = "Worker waiter #{Process.pid}"
+        while true
+          pid = Process.wait
+          remove_worker(pid)
+          break if @worker_mutex.synchronize{ @workers.empty? }
+        end
       end
-    end if @output
+    end
+
+    Thread.pass until @worker_mutex.synchronize{ @workers.select{|w| w.pid.nil? }.empty? }
+    Thread.pass until @waiter["name"]
   end
 
   def write(obj)
     @input.write obj
   end
 
+  def abort
+    Log.low "Aborting #{@workers.length} workers in queue #{Process.pid}"
+    @worker_mutex.synchronize do
+      @workers.each{|w| w.abort }
+    end
+  end
+
   def close
-    while @worker_mutex.synchronize{ @workers.length } > 0
-      begin
-        @input.write DoneProcessing.new
-        pid = Process.wait
-        status = $?
-        worker = @worker_mutex.synchronize{ @removed_workers.delete_if{|w| w.pid == pid }.first }
-        worker.exit $?.exitstatus if worker
-      rescue Errno::ECHILD
-        Thread.pass until @workers.length == 0
-        break
-      end
+    @closed = true
+    @worker_mutex.synchronize{ @workers.length }.times do
+      @input.write DoneProcessing.new()
     end
-    @reader.raise Aborted if @reader
   end
 
   def join
+    @waiter.join if @waiter
     @reader.join if @reader
   end
 end

data/lib/scout/workflow/definition.rb

@@ -2,7 +2,7 @@ require_relative '../meta_extension'
 
 module Workflow
   extend MetaExtension
-  extension_attr :name, :tasks
+  extension_attr :name, :tasks, :helpers
 
   class << self
     attr_accessor :directory
@@ -17,6 +17,32 @@ module Workflow
     @name ||= self.to_s
   end
 
+  def helpers
+    @helpers ||= {}
+  end
+
+  def helper(name, *args, &block)
+    if block_given?
+      helpers[name] = block
+    else
+      raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
+      helpers[name].call(*args)
+    end
+  end
+
+  def step_module
+    @_m ||= begin
+              m = Module.new
+
+              helpers.each do |name,block|
+                m.send(:define_method, name, &block)
+              end
+
+              m
+            end
+    @_m
+  end
+
   attr_accessor :directory
   def directory
     @directory ||= Workflow.directory[name]
@@ -60,19 +86,82 @@ module Workflow
     annotate_next_task(:inputs, args)
   end
 
+  def desc(description)
+    annotate_next_task_single(:description, description)
+  end
+
+  def returns(type)
+    annotate_next_task_single(:returns, type)
+  end
+
+  def extension(extension)
+    annotate_next_task_single(:extension, extension)
+  end
+
   def task(name_and_type, &block)
     name, type = name_and_type.collect.first
     @tasks ||= IndiferentHash.setup({})
     begin
       @annotate_next_task ||= {}
-      task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
+      task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
       @tasks[name] = task
     ensure
       @annotate_next_task = {}
     end
   end
 
-  def desc(description)
-    annotate_next_task_single(:description, description)
+  def task_alias(name, workflow, oname, *rest, &block)
+    dep(workflow, oname, *rest, &block) 
+    extension :dep_task unless @extension
+    returns workflow.tasks[oname].returns if workflow.tasks.include?(oname) unless @returns 
+    task name => nil do
+      raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
+      Step.wait_for_jobs dependencies.select{|d| d.streaming? }
+      dep = dependencies.last
+      dep.join
+      raise dep.get_exception if dep.error?
+      raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
+      set_info :result_type, dep.info[:result_type]
+      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
+      if forget
+        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
+
+        self.archive_deps
+        self.copy_files_dir
+        self.dependencies = self.dependencies - [dep]
+        Open.rm_rf self.files_dir if Open.exist? self.files_dir
+        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
+
+        if dep.overriden || ! Workflow.job_path?(dep.path)
+          Open.link dep.path, self.tmp_path
+        else
+          Open.ln_h dep.path, self.tmp_path
+
+          case remove.to_s
+          when 'true'
+            dep.clean
+          when 'recursive'
+            (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+              next if d.overriden
+              d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
+            end
+            dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
+          end 
+        end
+      else
+        if Open.exists?(dep.files_dir)
+          Open.rm_rf self.files_dir 
+          Open.link dep.files_dir, self.files_dir
+        end
+        if defined?(RemoteStep) && RemoteStep === dep
+          Open.write(self.tmp_path, Open.read(dep.path))
+        else
+          Open.link dep.path, self.path
+        end
+      end
+      nil
+    end
   end
+
+  alias dep_task task_alias
 end

data/lib/scout/workflow/step/config.rb

@@ -0,0 +1,18 @@
+require_relative '../../config'
+
+class Step
+  def config(key, *tokens)
+    options = tokens.pop if Hash === tokens.last
+    options ||= {}
+
+    new_tokens = []
+    if workflow
+      workflow_name = workflow.to_s
+      new_tokens << ("workflow:" << workflow_name)
+      new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
+    end
+    new_tokens << ("task:" << task_name.to_s)
+
+    Scout::Config.get(key, tokens + new_tokens, options)
+  end
+end

data/lib/scout/workflow/step/dependencies.rb

@@ -0,0 +1,40 @@
+class Step
+  def recursive_inputs
+    dependencies.inject(@inputs.annotate(@inputs.dup)) do |acc,dep|
+      acc.concat(dep.inputs) if dep.inputs
+      acc
+    end
+  end
+
+  def input_dependencies
+    return [] unless inputs
+    inputs.select do |d|
+      Step === d
+    end
+  end
+
+  def prepare_dependencies
+    inverse_dep = {}
+    dependencies.each{|dep| 
+      next if dep.done?
+      if dep.dependencies
+        dep.dependencies.each do |d|
+          inverse_dep[d] ||= []
+          inverse_dep[d] << dep 
+        end
+      end
+      input_dependencies.each do |d|
+        inverse_dep[d] ||= []
+        inverse_dep[d] << dep 
+      end
+    }
+    inverse_dep.each do |dep,list|
+      dep.tee_copies = list.length
+    end
+  end
+
+  def run_dependencies
+    dependencies.each{|dep| dep.run unless dep.running? || dep.done? }
+  end
+
+end

data/lib/scout/workflow/step/file.rb

@@ -0,0 +1,15 @@
+class Step
+  def files_dir
+    @files_dir ||= begin
+                     dir = @path + ".files"
+                     @path.annotate(dir) if Path === @path
+                     dir
+                   end
+  end
+
+  def file(file)
+    dir = files_dir
+    Path.setup(dir) unless Path === dir
+    dir[file]
+  end
+end

data/lib/scout/workflow/step/info.rb

@@ -1,5 +1,5 @@
 class Step
-  SERIALIZER = :json
+  SERIALIZER = :marshal
   def info_file
     @info_file ||= begin
                      info_file = @path + ".info"
@@ -20,7 +20,11 @@ class Step
   end
 
   def info
-    outdated = @info && Open.exists?(info_file) && @info_load_time && Open.mtime(info_file) > @info_load_time
+    outdated = begin
+                 @info_load_time && (mtime = Open.mtime(info_file)) && mtime > @info_load_time
+               rescue
+                 true
+               end
 
     if @info.nil? || outdated
       load_info
@@ -33,6 +37,19 @@ class Step
     info = self.info
     new_info.each do |key,value|
       report_status new_info[:status], new_info[:message] if key == :status
+      if Exception === value
+        begin
+          Marshal.dump(value)
+        rescue TypeError
+          if ScoutException === value
+            new = ScoutException.new value.message
+          else
+            new = Exception.new value.message
+          end
+          new.set_backtrace(value.backtrace)
+          value = new
+        end
+      end
       if info.include?(key)
         case info[key]
         when Array
@@ -55,14 +72,13 @@ class Step
   
   def report_status(status, message = nil)
     if message.nil?
-      Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path)
+      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path)
     else
-      Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path) + " " + message
+      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path) + " " + message
     end
   end
 
   def log(status, message = nil)
-    report_status status, message
     if message
       merge_info :status => status, :messages => [message]
     else
@@ -71,7 +87,18 @@ class Step
   end
 
   def status
-    info[:status]
+    info[:status].tap{|s| s.nil? ? s : s.to_sym }
+  end
+
+  def error?
+    status == :error
   end
 
+  def aborted?
+    status == :aborted
+  end
+
+  def running?
+    ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
+  end
 end

data/lib/scout/workflow/step/provenance.rb

@@ -0,0 +1,148 @@
+class Step
+  def self.job_path?(path)
+    path.split("/")[-4] == "jobs"
+  end
+
+  def self.status_color(status)
+    case status.to_sym
+    when :error, :aborted, :missing, :dead, :unsync
+      :red
+    when :streaming, :started
+      :cyan
+    when :done, :noinfo
+      :green
+    when :dependencies, :waiting, :setup
+      :yellow
+    when :notfound, :cleaned
+      :blue
+    else
+      if status.to_s.index ">"
+        :cyan
+      else
+        :cyan
+      end
+    end
+  end
+
+  def self.prov_status_msg(status)
+    color = status_color(status)
+    Log.color(color, status.to_s)
+  end
+
+  def self.prov_report_msg(status, name, path, info, input = nil)
+    parts = path.sub(/\{.*/,'').split "/"
+
+    parts.pop
+
+    task = Log.color(:yellow, parts.pop)
+    workflow = Log.color(:magenta, parts.pop)
+
+    if ! Step.job_path?(path)
+      task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
+    end
+
+    path_mtime = begin
+                   Open.mtime(path)
+                 rescue Exception
+                   nil
+                 end
+
+    if input.nil? || input.empty?
+      input_str = nil
+    else
+      input = input.reject{|dep,name| (input & dep.dependencies.collect{|d| [d,name]}).any? }
+      input = input.reject{|dep,name| (input & dep.input_dependencies.collect{|d| [d,name]}).any? }
+      input_str = Log.color(:magenta, "-> ") + input.collect{|dep,name| Log.color(:yellow, dep.task_name.to_s) + ":" + Log.color(:yellow, name) }.uniq * " "
+    end
+
+    str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
+          else
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " 
+          end
+
+    if $inputs and $inputs.any? 
+      job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
+      IndiferentHash.setup(job_inputs)
+
+      $inputs.each do |input|
+        value = job_inputs[input]
+        next if  value.nil?
+        value_str = Misc.fingerprint(value)
+        str << "\t#{Log.color :magenta, input}=#{value_str}"
+      end
+    end
+
+    if $info_fields and $info_fields.any?
+      $info_fields.each do |field|
+        IndiferentHash.setup(info)
+        value = info[field]
+        next if value.nil?
+        value_str = Misc.fingerprint(value)
+        str << "\t#{Log.color :magenta, field}=#{value_str}"
+      end
+    end
+
+    str << "\n"
+  end
+
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
+    info = step.info  || {}
+    info[:task_name] = task
+    path  = step.path
+    status = info[:status] || :missing
+    status = "remote" if Open.remote?(path) || Open.ssh?(path)
+    name = info[:name] || File.basename(path)
+    status = :unsync if status == :done and not Open.exist?(path)
+    status = :notfound if status == :noinfo and not Open.exist?(path)
+
+
+    this_step_msg = prov_report_msg(status, name, path, info, input)
+
+    input_dependencies = {}
+    step.dependencies.each do |dep|
+      if dep.input_dependencies.any?
+        dep.input_dependencies.each do |id|
+          input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+            d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
+          }.last
+          if input_name
+            input_dependencies[id] ||= []
+            input_dependencies[id] << [dep, input_name]
+          end
+        end
+      end
+    end if step.dependencies
+
+    str = ""
+    str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
+    step.dependencies.dup.tap{|l| 
+      l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    }.each do |dep|
+      path = dep.path
+      new = ! seen.include?(path)
+      if new
+        seen << path
+        str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
+      else
+        if expand_repeats
+          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
+        else
+          info = dep.info  || {}
+          status = info[:status] || :missing
+          status = "remote" if Open.remote?(path) || Open.ssh?(path)
+          name = info[:name] || File.basename(path)
+          status = :unsync if status == :done and not Open.exist?(path)
+          status = :notfound if status == :noinfo and not Open.exist?(path)
+
+          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
+        end
+      end
+    end if step.dependencies
+
+    str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
+    str
+  end
+end

data/lib/scout/workflow/step.rb

@@ -2,15 +2,25 @@ require_relative '../path'
 require_relative '../persist'
 require_relative 'step/info'
 require_relative 'step/load'
+require_relative 'step/file'
+require_relative 'step/dependencies'
+require_relative 'step/provenance'
+require_relative 'step/config'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :task
+  attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
   def initialize(path, inputs = nil, dependencies = nil, &task) 
     @path = path
     @inputs = inputs
     @dependencies = dependencies
     @task = task
+    @mutex = Mutex.new
+    @tee_copies = 1
+  end
+
+  def synchronize(&block)
+    @mutex.synchronize(&block)
   end
 
   def inputs
@@ -48,30 +58,49 @@ class Step
     @task_name ||= @task.name if @task.respond_to?(:name)
   end
 
+  def workflow
+    @task.workflow if @task
+  end
+
   def exec
-    self.instance_exec(*inputs, &task)
+    @result = self.instance_exec(*inputs, &task)
   end
 
   attr_reader :result
   def run
     return @result || self.load if done?
-    dependencies.each{|dep| dep.run }
-    @result = Persist.persist(name, type, :path => path) do
+    prepare_dependencies
+    run_dependencies
+    @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
       begin
         merge_info :status => :start, :start => Time.now,
           :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
           :inputs => inputs, :type => type,
           :dependencies => dependencies.collect{|d| d.path }
 
-        @result = exec
+        exec
+      rescue Exception => e
+        merge_info :status => :error, :exception => e
+        raise e
       ensure
-        if streaming?
-          ConcurrentStream.setup(@result) do
+        if ! (error? || aborted?)
+          if streaming?
+            ConcurrentStream.setup(@result) do
+              merge_info :status => :done, :end => Time.now
+            end
+
+            @result.abort_callback = proc do |exception|
+              if Aborted === exception || Interrupt === exception
+                merge_info :status => :aborted, :end => Time.now
+              else
+                merge_info :status => :error, :exception => exception, :end => Time.now
+              end
+            end
+
+            log :streaming
+          else
             merge_info :status => :done, :end => Time.now
           end
-          log :streaming
-        else
-          merge_info :status => :done, :end => Time.now
         end
       end
     end
@@ -82,19 +111,34 @@ class Step
   end
 
   def streaming?
-    IO === @result || StringIO === @result
+    @take_stream || IO === @result || StringIO === @result
   end
 
-  def stream
-    join
-    streaming? ? @result : Open.open(path)
+  def get_stream
+    synchronize do
+      if streaming? && ! @result.nil?
+        if @result.next
+          Log.debug "Taking result #{Log.fingerprint @result} next #{Log.fingerprint @result.next}"
+        else
+          Log.debug "Taking result #{Log.fingerprint @result}"
+        end
+        @take_stream, @result = @result, @result.next
+        @take_stream
+      elsif done?
+        Open.open(self.path)
+      else
+        if running?
+          nil
+        else
+          exec
+        end
+      end
+    end
   end
 
   def join
-    if streaming?
-      Open.consume_stream(@result, false) 
-      @result = nil
-    end
+    stream = get_stream if streaming?
+    Open.consume_stream(stream, false) if stream
   end
 
   def produce
@@ -109,7 +153,13 @@ class Step
   end
 
   def clean
-    FileUtils.rm path.find if path.exist?
+    @take_stream = nil 
+    @result = nil
+    @info = nil
+    @info_load_time = nil
+    Open.rm path if Open.exist?(path)
+    Open.rm info_file if Open.exist?(info_file)
+    Open.rm_rf files_dir if Open.exist?(files_dir)
   end
 
   def recursive_clean
@@ -127,6 +177,6 @@ class Step
   end
 
   def digest_str
-    path
+    path.dup
   end
 end