scout-gear 10.4.0 → 10.7.0

data/lib/scout/workflow/step/info.rb

@@ -19,15 +19,17 @@ class Step
     @info_load_time = Time.now
   end
 
-
-
   def save_info(info = nil)
     Persist.save(info, info_file, SERIALIZER)
     @info_load_time = Time.now
   end
 
-  def clear_info
-    save_info(@info = {})
+  def reset_info(info = {})
+    save_info(@info = info)
+  end
+
+  def init_info
+    log :waiting unless info_file.nil? || Open.exists?(info_file)
   end
 
   def info
@@ -47,20 +49,36 @@ class Step
   def merge_info(new_info)
     info = self.info
     new_info.each do |key,value|
+      value = Annotation.purge(value)
       if key == :status
-        message = new_info[:messages]
+        message = new_info[:message]
         if message.nil? && (value == :done || value == :error || value == :aborted)
+          issued = info[:issued]
           start = info[:start]
           eend = new_info[:end]
           if start && eend
             time = eend - start
-            time_str = Misc.format_seconds_short(time)
+            total_time = eend - issued
+            if total_time - time > 1
+              time_str = "#{Misc.format_seconds_short(time)} (#{Misc.format_seconds_short(total_time)})"
+            else
+              time_str = Misc.format_seconds_short(time)
+            end
+            info[:time_elapsed] = time
+            info[:total_time_elapsed] = total_time
             message = Log.color(:time, time_str)
           end
         end
         report_status value, message 
       end
 
+      if key == :message
+        messages = info[:messages] || []
+        messages << value
+        info[:messages] = messages
+        next
+      end
+
       if Exception === value
         begin
           Marshal.dump(value)
@@ -111,12 +129,16 @@ class Step
     end
 
     if message
-      merge_info :status => status, :messages => message
+      merge_info :status => status, :message => message
     else
       merge_info :status => status
     end
   end
 
+  def messages
+    info[:messages]
+  end
+
   def status
     info[:status].tap{|s| s.nil? ? s : s.to_sym }
   end
@@ -134,7 +156,8 @@ class Step
   end
 
   def overriden?
-    overriden_task || overriden_workflow || dependencies.select{|d| d.overriden? }.any?
+    @overriden = overriden_task || overriden_workflow || overriden_deps.any? if @overriden.nil?
+    @overriden
   end
 
   def overriden_deps
@@ -145,10 +168,15 @@ class Step
     info[:exception]
   end
 
-  def marshal_dump
+  # Marshal Step
+  def _dump(level)
     @path
   end
 
+  def self._load(path)
+    Step.new path
+  end
+
   def marshal_load(path)
     Step.new path
   end

data/lib/scout/workflow/step/progress.rb

@@ -1,5 +1,5 @@
 class Step
-  def progress_bar(msg = "Progress", options = nil)
+  def progress_bar(msg = "Progress", options = nil, &block)
     if Hash === msg and options.nil?
       options = msg
       msg = nil
@@ -8,7 +8,16 @@ class Step
 
     max = options[:max]
     Open.mkdir files_dir
-    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
+    bar = Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
+
+    if block_given?
+      bar.init
+      res = yield bar
+      bar.remove
+      res
+    else
+      bar
+    end
   end
 end
 

data/lib/scout/workflow/step/status.rb

@@ -1,11 +1,18 @@
 class Step
   def abort(exception = nil)
     if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
-      Process.kill pid
+      Log.debug "Kill process #{pid} to abort step #{Log.fingerprint self}"
+      begin
+        s = Misc.abort_child pid, true
+        Log.medium "Aborted pid #{path} #{s}"
+      rescue 
+        Log.debug("Aborted job #{pid} was not killed: #{$!.message}")
+      end
     else
       while @result && streaming? && stream = self.stream
         stream.abort(exception)
       end
+      @take_stream.abort(exception) if streaming?
     end
   end
 
@@ -19,6 +26,7 @@ class Step
     newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
 
+    Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
     newer.empty?
   end
 

data/lib/scout/workflow/step.rb

@@ -1,5 +1,6 @@
 require 'scout/path'
 require 'scout/persist'
+require 'scout/semaphore'
 require_relative 'step/info'
 require_relative 'step/status'
 require_relative 'step/load'
@@ -9,11 +10,13 @@ require_relative 'step/provenance'
 require_relative 'step/config'
 require_relative 'step/progress'
 require_relative 'step/inputs'
+require_relative 'step/children'
+require_relative 'step/archive'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
+  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context = nil, &task)
     @path = path
     @inputs = inputs
     @dependencies = dependencies
@@ -24,6 +27,7 @@ class Step
     @task = task
     @mutex = Mutex.new
     @tee_copies = 1
+    @exec_context = exec_context || self
   end
 
   def synchronize(&block)
@@ -44,10 +48,11 @@ class Step
 
   def dependencies
     @dependencies ||= begin
-                        if Open.exists?(info_file)
+                        if info_file && Open.exists?(info_file) && info[:dependencies]
                           info[:dependencies].collect do |path|
+                            path = path.last if Array === path
                             Step.load(path)
-                          end if info[:dependencies]
+                          end
                         else
                           []
                         end
@@ -56,7 +61,7 @@ class Step
 
   attr_accessor :type
   def type
-    @type ||= @task.respond_to?(:type) ? @task.type : info[:type]
+    @type ||= (@task.respond_to?(:type) && @task.type) ? @task.type : info[:type]
   end
 
   def name
@@ -70,7 +75,9 @@ class Step
   def clean_name
     return @id if @id
     return info[:clean_name] if info.include? :clean_name
-    return m[1] if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
+    if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
+      return m[1] 
+    end
     return name.split(".").first
   end
 
@@ -82,10 +89,16 @@ class Step
 
   def workflow
     @workflow ||= @task.workflow if Task === @task
-    @workflow ||= info[:workflow] if Open.exist?(info_file)
+    @workflow ||= info[:workflow] if info_file && Open.exist?(info_file)
     @workflow ||= path.split("/")[-3]
   end
 
+  def full_task_name
+    return nil if task_name.nil?
+    return task_name.to_s if workflow.nil?
+    [workflow, task_name] * "#"
+  end
+
   def exec
 
     if inputs 
@@ -107,7 +120,7 @@ class Step
 
     @result = begin
                 @in_exec = true
-                self.instance_exec(*inputs, &task)
+                @exec_context.instance_exec(*inputs, &task)
               ensure
                 @in_exec = false
               end
@@ -127,23 +140,37 @@ class Step
   def run(stream = false)
     return @result || self.load if done?
     prepare_dependencies
-    run_dependencies
     begin
-      @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
-        clear_info
+
+      case stream
+      when TrueClass, :stream
+        no_load = :stream
+      when :no_load
+        no_load = true
+      else
+        no_load = false
+      end
+
+      @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies, no_load: no_load) do
         input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
-        merge_info :status => :start, :start => Time.now,
+
+
+        reset_info :status => :setup, :issued => Time.now,
           :pid => Process.pid, :pid_hostname => Misc.hostname, 
           :task_name => task_name, :workflow => workflow.to_s,
-          :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
-          :dependencies => dependencies.collect{|d| d.path }
+          :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
+          :dependencies => (dependencies || []) .collect{|d| d.path }
 
+        run_dependencies
+
+        set_info :start, Time.now
+        log :start
         @exec_result = exec
 
         if @exec_result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
           Open.mv self.tmp_path, self.path
         else
-          @exec_result = @exec_result.stream if @exec_result.respond_to?(:stream)
+          @exec_result = @exec_result.stream if @exec_result.respond_to?(:stream) && ! (TSV === @exec_result)
         end
 
         @exec_result
@@ -155,13 +182,25 @@ class Step
           @exec_result
         end
       end
+
+      if TrueClass === no_load
+        consume_all_streams if streaming?
+        @result = nil
+      elsif no_load && ! (IO === @result)
+        @result = nil
+      end
+
+      @result
     rescue Exception => e
-      merge_info :status => :error, :exception => e, :end => Time.now
-      abort_dependencies
-      raise e
+      merge_info :status => :error, :exception => e, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
+      begin
+        abort_dependencies
+      ensure
+        raise e
+      end
     ensure
       if ! (error? || aborted?)
-        if streaming?
+        if @result && streaming?
           ConcurrentStream.setup(@result) do
             merge_info :status => :done, :end => Time.now
           end
@@ -187,12 +226,22 @@ class Step
     end
   end
 
-  def fork
-    Process.fork do
-      clear_info unless present?
-      run(false)
+  def fork(noload = false, semaphore = nil)
+    pid = Process.fork do
+      Signal.trap(:TERM) do
+        raise Aborted, "Recieved TERM Signal on forked process #{Process.pid}"
+      end
+      reset_info status: :queue, pid: Process.pid unless present?
+      if semaphore
+        ScoutSemaphore.synchronize(semaphore) do
+          run(noload)
+        end
+      else
+        run(noload)
+      end
       join
     end
+    Process.detach pid
     grace
     self
   end
@@ -272,17 +321,23 @@ class Step
     while @result.nil? && (present? && ! (terminated? || done?))
       sleep 0.1
     end
+
+    Misc.wait_child info[:pid] if info[:pid]
+
     raise self.exception if self.exception
+
     raise "Error in job #{self.path}" if self.error? or self.aborted? 
+
     self
   end
 
   def produce(with_fork: false)
+    clean if error? && recoverable_error?
     if with_fork
       self.fork
       self.join
     else
-      run
+      run(:no_load)
     end
     self
   end
@@ -305,7 +360,7 @@ class Step
   end
 
   def short_path
-    Scout.identify @path
+    Resource.identify @path
   end
 
   def digest_str

data/lib/scout/workflow/task/dependencies.rb

@@ -19,7 +19,7 @@ module Task
       step_inputs.each do |k,v|
         if Symbol === v
           input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || k
+          resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || v
         else
           resolved_inputs[k] = v
         end
@@ -31,6 +31,9 @@ module Task
       compute_options << :produce if definition_options[:produce]
       compute_options << :stream if definition_options[:stream]
       compute[job.path] = compute_options if compute_options.any?
+
+      job.overriden = false if definition_options[:not_overriden]
+
       [job, step_inputs]
     end
 
@@ -98,7 +101,7 @@ module Task
             else
               dep = _res
               dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
               non_default_inputs.concat(dep_non_default_inputs)
             end
           end

data/lib/scout/workflow/task/inputs.rb

@@ -40,7 +40,16 @@ module Task
       input_names << name
       provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
       provided = Task.format_input(provided, type, options || {})
-      if ! provided.nil? && provided != value
+      if provided == value
+        same_as_default = true
+      elsif String === provided && Symbol === value && provided == value.to_s
+        same_as_default = true
+      elsif String === value && Symbol === provided && provided.to_s == value
+        same_as_default = true
+      else
+        same_as_default = false
+      end
+      if ! provided.nil? && ! same_as_default
         non_default_inputs << name.to_sym
         input_array << provided
       elsif options && options[:jobname]
@@ -61,7 +70,7 @@ module Task
     [input_array, non_default_inputs, digest_str]
   end
 
-  def save_file_input(orig_file, directory)
+  def self.save_file_input(orig_file, directory)
     orig_file = orig_file.path if Step === orig_file
     basename = File.basename(orig_file)
     digest = Misc.digest(orig_file)
@@ -76,38 +85,65 @@ module Task
     relative_file
   end
 
+  def self.save_input(directory, name, type, value)
+    input_file = File.join(directory, name.to_s)
+
+    if Path.is_filename?(value) 
+      if type == :path
+        Open.write(input_file + ".as_path", value)
+      else
+        relative_file = save_file_input(value, directory)
+        Open.write(input_file + ".as_file", relative_file)
+      end
+    elsif Step === value
+      Open.write(input_file + ".as_step", value.short_path)
+    elsif type == :file
+      relative_file = save_file_input(value, directory)
+      Persist.save(relative_file, input_file, :file)
+    elsif type == :file_array
+      new_files = value.collect do |orig_file|
+        save_file_input(orig_file, directory)
+      end
+      Persist.save(new_files, input_file, type)
+    elsif Open.is_stream?(value)
+      Open.sensible_write(input_file, value)
+    elsif Open.has_stream?(value)
+      Open.sensible_write(input_file, value.stream)
+    else
+      Persist.save(value, input_file, type)
+    end
+  end
+
   def save_inputs(directory, provided_inputs = {})
-    #input_array, non_default_inputs = assign_inputs(provided_inputs)
     self.recursive_inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
       next unless provided_inputs.include?(name)
       value = provided_inputs[name]
-      input_file = File.join(directory, name.to_s)
-
-      if Path.is_filename?(value) 
-        if type == :path
-          Open.write(input_file + ".as_path", value)
-        else
-          relative_file = save_file_input(value, directory)
-          Open.write(input_file + ".as_file", relative_file)
-        end
-      elsif Step === value
-        Open.write(input_file + ".as_step", value.short_path)
-      elsif type == :file
-        relative_file = save_file_input(value, directory)
-        Persist.save(relative_file, input_file, :file)
-      elsif type == :file_array
-        new_files = value.collect do |orig_file|
-          save_file_input(orig_file, directory)
-        end
-        Persist.save(new_files, input_file, type)
-      elsif Open.is_stream?(value)
-        Persist.save(value, input_file, type)
-      elsif Open.has_stream?(value)
-        Persist.save(value.stream, input_file, type)
+
+      Task.save_input(directory, name, type, value)
+    end
+  end
+
+
+  def self.load_input_from_file(filename, type, options = nil)
+    if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
+      if filename.end_with?('.as_file')
+        value = Open.read(filename).strip
+        value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
+        value
+      elsif filename.end_with?('.as_step')
+        value = Open.read(filename).strip
+        Step.load value
+      elsif filename.end_with?('.as_path')
+        value = Open.read(filename).strip
+        Path.setup value
+      elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+        filename
       else
-        Persist.save(value, input_file, type)
+        Persist.load(filename, type)
       end
+    else
+      return nil
     end
   end
 
@@ -116,21 +152,35 @@ module Task
     self.recursive_inputs.each do |p|
       name, type, desc, value, options = p
       filename = File.join(directory, name.to_s) 
-      if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
-        if filename.end_with?('.as_file')
-          value = Open.read(filename).strip
-          value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
-          inputs[name] = value
-        elsif filename.end_with?('.as_step')
-          value = Open.read(filename).strip
-          inputs[name] = Step.load value
-        elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
-          inputs[name] = filename
-        else
-          inputs[name] = Persist.load(filename, type)
-        end
-      end
+      value = Task.load_input_from_file(filename, type, options)
+      inputs[name] = value unless value.nil?
+    end
+
+    Dir.glob(File.join(directory, "*#*")).each do |file|
+      override_dep, _, extension = File.basename(file).partition(".")
+
+      inputs[override_dep] = Task.load_input_from_file(file, :file)
     end
+
     inputs
   end
+
+  def recursive_inputs(overriden = [])
+    return inputs.dup if deps.nil?
+    deps.inject(inputs.dup) do |acc,dep|
+      workflow, task, options = dep
+      next acc if workflow.nil? || task.nil?
+      next acc if overriden.include?([workflow.name, task.to_s] * "#")
+      overriden.concat options.keys.select{|k| k.to_s.include?("#") } if options
+
+      workflow.tasks[task].recursive_inputs(overriden).dup.each do |info|
+        name, _ = info
+        next if options.include?(name.to_sym) || options.include?(name.to_s)
+        acc << info
+      end
+
+      acc
+    end
+  end
+
 end