scout-gear 7.2.0 → 8.0.0

data/lib/scout/work_queue.rb

@@ -46,7 +46,11 @@ class WorkQueue
   end
 
   def process(&callback)
-    @reader = Thread.new do |parent|
+    @workers.each do |w| 
+      w.process @input, @output, &@worker_proc
+    end
+
+    @reader = Thread.new(Thread.current) do |parent|
       begin
         Thread.current.report_on_exception = false
         Thread.current["name"] = "Output reader #{Process.pid}"
@@ -71,8 +75,9 @@ class WorkQueue
       rescue DoneProcessing
       rescue Aborted
       rescue WorkerException
-        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
+        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.worker_exception.message}"
         self.abort
+        @input.abort obj.worker_exception
         raise obj.worker_exception
       rescue
         Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
@@ -81,10 +86,6 @@ class WorkQueue
       end
     end
 
-    @workers.each do |w| 
-      w.process @input, @output, &@worker_proc
-    end
-
     Thread.pass until @reader["name"]
 
     @waiter = Thread.new do
@@ -104,7 +105,13 @@ class WorkQueue
   end
 
   def write(obj)
-    @input.write obj
+    begin
+      @input.write obj
+    rescue Exception
+      raise $! unless @input.exception
+    ensure
+      raise @input.exception if @input.exception
+    end
   end
 
   def abort
@@ -117,7 +124,7 @@ class WorkQueue
   def close
     @closed = true
     @worker_mutex.synchronize{ @workers.length }.times do
-      @input.write DoneProcessing.new()
+      @input.write DoneProcessing.new() unless @input.closed_write?
     end
   end
 

data/lib/scout/workflow/definition.rb

@@ -76,9 +76,11 @@ module Workflow
       end
     when 1
       task = args.first
+      options, task = task, nil if Hash === task
     end
     workflow = self if workflow.nil?
     options = {} if options.nil?
+    task = task.to_sym if task
     annotate_next_task :deps, [workflow, task, options, block, args]
   end
 
@@ -103,6 +105,20 @@ module Workflow
     @tasks ||= IndiferentHash.setup({})
     begin
       @annotate_next_task ||= {}
+      @annotate_next_task[:extension] ||=  
+        case type
+        when :tsv
+          "tsv"
+        when :yaml
+          "yaml"
+        when :marshal
+          "marshal"
+        when :json
+          "json"
+        else
+          nil
+        end
+
       task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
       @tasks[name] = task
     ensure
@@ -110,11 +126,14 @@ module Workflow
     end
   end
 
+  FORGET_DEP_TASKS = ENV["SCOUT_FORGET_DEP_TASKS"] == "true"
+  REMOVE_DEP_TASKS = ENV["SCOUT_REMOVE_DEP_TASKS"] == "true"
   def task_alias(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
-    returns workflow.tasks[oname].returns if workflow.tasks.include?(oname) unless @returns 
-    task name => nil do
+    returns workflow.tasks[oname].returns if @returns.nil?
+    type = workflow.tasks[oname].type 
+    task name => type do
       raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
@@ -164,4 +183,12 @@ module Workflow
   end
 
   alias dep_task task_alias
+
+  def export(*args)
+  end
+
+  alias export_synchronous export
+  alias export_asynchronous export
+  alias export_exec export
+  alias export_stream export
 end

data/lib/scout/workflow/step/dependencies.rb

@@ -1,8 +1,13 @@
 class Step
+  def rec_dependencies
+    rec_dependencies = dependencies.dup
+    dependencies.inject(rec_dependencies){|acc,d| acc.concat d.rec_dependencies }
+  end
+
   def recursive_inputs
-    dependencies.inject(@inputs.annotate(@inputs.dup)) do |acc,dep|
-      acc.concat(dep.inputs) if dep.inputs
-      acc
+    recursive_inputs = @inputs.to_hash
+    dependencies.inject(recursive_inputs) do |acc,dep|
+      acc.merge(dep.recursive_inputs)
     end
   end
 
@@ -16,6 +21,10 @@ class Step
   def prepare_dependencies
     inverse_dep = {}
     dependencies.each{|dep| 
+      if dep.present? && ! dep.updated? 
+        Log.debug "Clean outdated #{dep.path}"
+        dep.clean
+      end
       next if dep.done?
       if dep.dependencies
         dep.dependencies.each do |d|
@@ -34,7 +43,18 @@ class Step
   end
 
   def run_dependencies
-    dependencies.each{|dep| dep.run unless dep.running? || dep.done? }
+    dependencies.each{|dep| dep.run(true) unless dep.running? || dep.done? }
   end
 
+  def abort_dependencies
+    dependencies.each{|dep| dep.abort if dep.running? }
+  end
+
+  def self.wait_for_jobs(jobs)
+    threads = []
+    jobs.each do |job| 
+      threads << job.join
+    end
+    threads.each do |t| t.join end
+  end
 end

data/lib/scout/workflow/step/info.rb

@@ -1,6 +1,7 @@
 class Step
   SERIALIZER = :marshal
   def info_file
+    return nil if @path.nil?
     @info_file ||= begin
                      info_file = @path + ".info"
                      @path.annotate info_file if Path === @path
@@ -19,6 +20,10 @@ class Step
     @info_load_time = Time.now
   end
 
+  def clear_info
+    save_info(@info = {})
+  end
+
   def info
     outdated = begin
                  @info_load_time && (mtime = Open.mtime(info_file)) && mtime > @info_load_time
@@ -36,7 +41,19 @@ class Step
   def merge_info(new_info)
     info = self.info
     new_info.each do |key,value|
-      report_status new_info[:status], new_info[:message] if key == :status
+      if key == :status
+        message = new_info[:messages]
+        if message.nil? && value == :done || value == :error || value == :aborted
+          start = info[:start]
+          eend = new_info[:end]
+          if start && eend
+            time = eend - start
+            time_str = Misc.format_seconds_short(time)
+            message = Log.color(:time, time_str)
+          end
+        end
+        report_status value, message 
+      end
       if Exception === value
         begin
           Marshal.dump(value)
@@ -72,15 +89,21 @@ class Step
   
   def report_status(status, message = nil)
     if message.nil?
-      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path)
+      Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
     else
-      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path) + " " + message
+      Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
     end
   end
 
-  def log(status, message = nil)
+  def log(status, message = nil, &block)
+    if block_given?
+      time = Misc.exec_time &block
+      time_str = Misc.format_seconds_short time
+      message = message.nil? ? Log.color(:time, time_str) : "#{Log.color :time, time_str} - #{ message }"
+    end
+
     if message
-      merge_info :status => status, :messages => [message]
+      merge_info :status => status, :messages => message
     else
       merge_info :status => status
     end
@@ -101,4 +124,16 @@ class Step
   def running?
     ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
   end
+
+  def exception
+    info[:exception]
+  end
+
+  def marshal_dump
+    @path
+  end
+
+  def marshal_load(path)
+    Step.new path
+  end
 end

data/lib/scout/workflow/step/progress.rb

@@ -0,0 +1,14 @@
+class Step
+  def progress_bar(msg = "Progress", options = nil)
+    if Hash === msg and options.nil?
+      options = msg
+      msg = nil
+    end
+    options = {} if options.nil?
+
+    max = options[:max]
+    Open.mkdir files_dir
+    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
+  end
+end
+

data/lib/scout/workflow/step/provenance.rb

@@ -5,7 +5,7 @@ class Step
 
   def self.status_color(status)
     case status.to_sym
-    when :error, :aborted, :missing, :dead, :unsync
+    when :error, :aborted, :dead, :unsync
       :red
     when :streaming, :started
       :cyan
@@ -13,7 +13,7 @@ class Step
       :green
     when :dependencies, :waiting, :setup
       :yellow
-    when :notfound, :cleaned
+    when :notfound, :cleaned, :missing
       :blue
     else
       if status.to_s.index ">"
@@ -91,6 +91,7 @@ class Step
     info[:task_name] = task
     path  = step.path
     status = info[:status] || :missing
+    status = :noinfo if status == :missing && Open.exist?(path)
     status = "remote" if Open.remote?(path) || Open.ssh?(path)
     name = info[:name] || File.basename(path)
     status = :unsync if status == :done and not Open.exist?(path)
@@ -103,9 +104,9 @@ class Step
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
-          input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+          input_name, _dep = dep.recursive_inputs.select{|f,d| 
             d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
-          }.last
+          }.keys.last
           if input_name
             input_dependencies[id] ||= []
             input_dependencies[id] << [dep, input_name]
@@ -115,10 +116,10 @@ class Step
     end if step.dependencies
 
     str = ""
-    str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    str = " " * offset + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
     step.dependencies.dup.tap{|l| 
-      l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+      l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
     }.each do |dep|
       path = dep.path
       new = ! seen.include?(path)
@@ -141,7 +142,7 @@ class Step
       end
     end if step.dependencies
 
-    str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    str += (" " * offset) + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
     str
   end

data/lib/scout/workflow/step/status.rb

@@ -0,0 +1,45 @@
+class Step
+  def abort(exception = nil)
+    while @result && streaming? && stream = self.stream
+      stream.abort(exception)
+    end
+  end
+
+  def recoverable_error?
+    self.error? && ! (ScoutException === self.exception)
+  end
+
+  def updated?
+    return false if self.error? && self.recoverable_error?
+    return true unless ENV["SCOUT_UPDATE"]
+    newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
+    newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
+
+    newer.empty?
+  end
+
+  def clean
+    @take_stream = nil 
+    @result = nil
+    @info = nil
+    @info_load_time = nil
+    Open.rm path if Open.exist?(path)
+    Open.rm info_file if Open.exist?(info_file)
+    Open.rm_rf files_dir if Open.exist?(files_dir)
+  end
+
+  def present?
+    Open.exist?(path) &&
+      Open.exist?(info_file) &&
+      Open.exist?(files_dir)
+  end
+
+
+  def recursive_clean
+    dependencies.each do |dep|
+      dep.recursive_clean
+    end
+    clean
+  end
+
+end

data/lib/scout/workflow/step.rb

@@ -1,19 +1,23 @@
 require_relative '../path'
 require_relative '../persist'
 require_relative 'step/info'
+require_relative 'step/status'
 require_relative 'step/load'
 require_relative 'step/file'
 require_relative 'step/dependencies'
 require_relative 'step/provenance'
 require_relative 'step/config'
+require_relative 'step/progress'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
-  def initialize(path, inputs = nil, dependencies = nil, &task) 
+  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, &task) 
     @path = path
     @inputs = inputs
     @dependencies = dependencies
+    @id = id
+    @non_default_inputs = non_default_inputs
     @task = task
     @mutex = Mutex.new
     @tee_copies = 1
@@ -25,7 +29,7 @@ class Step
 
   def inputs
     @inputs ||= begin
-                  if Open.exists?(info_file)
+                  if info_file && Open.exists?(info_file)
                     info[:inputs]
                   else
                     []
@@ -54,6 +58,13 @@ class Step
     @name ||= File.basename(@path)
   end
 
+  def clean_name
+    return @id if @id
+    return info[:clean_name] if info.include? :clean_name
+    return m[1] if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
+    return name.split(".").first
+  end
+
   def task_name
     @task_name ||= @task.name if @task.respond_to?(:name)
   end
@@ -63,24 +74,65 @@ class Step
   end
 
   def exec
+
+    if inputs 
+      if Task === task
+        types = task.inputs.collect{|name,type| type }
+        new_inputs = inputs.zip(types).collect{|input,info|  
+          type, desc, default, options = info
+          next input unless Step === input
+          input.join if input.streaming?
+          Task.format_input(input.join.path, type, options)
+        }
+      else
+        new_inputs = inputs.collect{|input|  
+          Step === input ? input.load : input
+        }
+      end
+      inputs = new_inputs
+    end
+
     @result = self.instance_exec(*inputs, &task)
   end
 
+  def tmp_path
+    @tmp_path ||= begin
+                    basename = File.basename(@path)
+                    dirname = File.dirname(@path)
+                    tmp_path = File.join(dirname, '.' + basename)
+                    @path.setup(tmp_path) if Path === @path
+                    tmp_path
+                  end
+  end
+
   attr_reader :result
-  def run
+  def run(stream = false)
     return @result || self.load if done?
     prepare_dependencies
     run_dependencies
-    @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
+    @result = 
       begin
-        merge_info :status => :start, :start => Time.now,
-          :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
-          :inputs => inputs, :type => type,
-          :dependencies => dependencies.collect{|d| d.path }
+        Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
+          clear_info
+          merge_info :status => :start, :start => Time.now,
+            :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
+            :inputs => inputs, :type => type,
+            :dependencies => dependencies.collect{|d| d.path }
+
+
+          @result = exec
 
-        exec
+          if @result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
+            Open.mv self.tmp_path, self.path
+          else
+            @result = @result.stream if @result.respond_to?(:stream)
+          end
+
+          @result
+        end
       rescue Exception => e
-        merge_info :status => :error, :exception => e
+        merge_info :status => :error, :exception => e, :end => Time.now
+        abort_dependencies
         raise e
       ensure
         if ! (error? || aborted?)
@@ -103,6 +155,17 @@ class Step
           end
         end
       end
+
+    if stream && ENV["SCOUT_NO_STREAM"].nil?
+      @result
+    else
+      if IO === @result || @result.respond_to?(:stream)
+        join
+        @result = nil
+        self.load
+      else
+        @result
+      end
     end
   end
 
@@ -111,10 +174,10 @@ class Step
   end
 
   def streaming?
-    @take_stream || IO === @result || StringIO === @result
+    @take_stream || IO === @result || StringIO === @result 
   end
 
-  def get_stream
+  def stream
     synchronize do
       if streaming? && ! @result.nil?
         if @result.next
@@ -136,9 +199,24 @@ class Step
     end
   end
 
+  def consume_all_streams
+    threads = [] 
+    while @result && streaming? && stream = self.stream
+      threads << Open.consume_stream(stream, true)
+    end
+    threads.each do |t| 
+      begin
+        t.join 
+      rescue
+        threads.each{|t| t.raise(Aborted); t.join }
+        raise $!
+      end
+    end
+  end
+
   def join
-    stream = get_stream if streaming?
-    Open.consume_stream(stream, false) if stream
+    consume_all_streams
+    self
   end
 
   def produce
@@ -152,31 +230,24 @@ class Step
     done? ? Persist.load(path, type) : exec
   end
 
-  def clean
-    @take_stream = nil 
-    @result = nil
-    @info = nil
-    @info_load_time = nil
-    Open.rm path if Open.exist?(path)
-    Open.rm info_file if Open.exist?(info_file)
-    Open.rm_rf files_dir if Open.exist?(files_dir)
-  end
-
-  def recursive_clean
-    dependencies.each do |dep|
-      dep.recursive_clean
-    end
-    clean
-  end
-
   def step(task_name)
     dependencies.each do |dep|
       return dep if dep.task_name == task_name
+      rec_dep = dep.step(task_name)
+      return rec_dep if rec_dep
     end
     nil
   end
 
+  def short_path
+    Scout.identify @path
+  end
+
   def digest_str
-    path.dup
+    "Step: " + short_path
+  end
+
+  def fingerprint
+    digest_str
   end
 end

data/lib/scout/workflow/task/inputs.rb

@@ -1,11 +1,10 @@
+require_relative '../../named_array'
 module Task
+  def self.format_input(value, type, options = {})
+    return value if IO === value || StringIO === value || Step === value
 
-  def format_input(value, type, options = {})
-    return value if IO === value || StringIO === value
-
-    value = value.load if Step === value
-    if String === value && ! [:path, :file].include?(type) 
-      if Open.exists?(value)
+    if String === value && ! [:path, :file, :folder].include?(type) && (options.nil? || (! options[:noload] || options[:stream]))
+      if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else
         Persist.deserialize(value, type)
@@ -13,7 +12,7 @@ module Task
     else
       if m = type.to_s.match(/(.*)_array/)
         if Array === value
-          value.collect{|v| format_input(v, m[1].to_sym, options) }
+          value.collect{|v| self.format_input(v, m[1].to_sym, options) }
         end
       else
         value
@@ -34,35 +33,30 @@ module Task
     IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
 
     input_array = []
+    input_names = []
     non_default_inputs = []
     self.inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
+      input_names << name
       provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
-      provided = format_input(provided, type, options || {})
+      provided = Task.format_input(provided, type, options || {})
       if ! provided.nil? && provided != value
-        non_default_inputs << name
+        non_default_inputs << name.to_sym
         input_array << provided
       else
         input_array << value
       end
     end
 
+    NamedArray.setup(input_array, input_names)
+
     [input_array, non_default_inputs]
   end
 
-  def digest_inputs(provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs(provided_inputs)
-    if Array === provided_inputs 
-      Misc.digest(input_array)
-    else
-      Misc.digest(input_array)
-    end
-  end
-  
   def process_inputs(provided_inputs = {})
     input_array, non_default_inputs = assign_inputs provided_inputs
-    digest = Misc.digest(input_array)
-    [input_array, non_default_inputs, digest]
+    digest_str = Misc.digest_str(input_array)
+    [input_array, non_default_inputs, digest_str]
   end
 
   def save_file_input(orig_file, directory)