scout-gear 8.0.0 → 8.1.0

data/lib/scout/workflow/step.rb

@@ -11,13 +11,14 @@ require_relative 'step/progress'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, &task) 
+  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :compute
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, compute = nil, &task) 
     @path = path
     @inputs = inputs
     @dependencies = dependencies
     @id = id
     @non_default_inputs = non_default_inputs
+    @compute = compute 
     @task = task
     @mutex = Mutex.new
     @tee_copies = 1
@@ -42,7 +43,7 @@ class Step
                         if Open.exists?(info_file)
                           info[:dependencies].collect do |path|
                             Step.load(path)
-                          end
+                          end if info[:dependencies]
                         else
                           []
                         end
@@ -92,7 +93,12 @@ class Step
       inputs = new_inputs
     end
 
-    @result = self.instance_exec(*inputs, &task)
+    @result = begin
+                @in_exec = true
+                self.instance_exec(*inputs, &task)
+              ensure
+                @in_exec = false
+              end
   end
 
   def tmp_path
@@ -115,11 +121,10 @@ class Step
         Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
           clear_info
           merge_info :status => :start, :start => Time.now,
-            :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
-            :inputs => inputs, :type => type,
+            :pid => Process.pid, :pid_hostname => Misc.hostname, 
+            :inputs => MetaExtension.purge(inputs), :type => type,
             :dependencies => dependencies.collect{|d| d.path }
 
-
           @result = exec
 
           if @result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
@@ -129,6 +134,13 @@ class Step
           end
 
           @result
+
+          if (IO === @result || StringIO === @result) && (ENV["SCOUT_NO_STREAM"] == "true" || ! stream)
+            Open.sensible_write(self.path, @result)
+            @result = nil
+          else
+            @result
+          end
         end
       rescue Exception => e
         merge_info :status => :error, :exception => e, :end => Time.now
@@ -142,10 +154,14 @@ class Step
             end
 
             @result.abort_callback = proc do |exception|
-              if Aborted === exception || Interrupt === exception
+              if exception.nil? || Aborted === exception || Interrupt === exception
                 merge_info :status => :aborted, :end => Time.now
               else
-                merge_info :status => :error, :exception => exception, :end => Time.now
+                begin
+                  merge_info :status => :error, :exception => exception, :end => Time.now
+                rescue Exception
+                  Log.exception $!
+                end
               end
             end
 
@@ -155,18 +171,16 @@ class Step
           end
         end
       end
+  end
 
-    if stream && ENV["SCOUT_NO_STREAM"].nil?
-      @result
-    else
-      if IO === @result || @result.respond_to?(:stream)
-        join
-        @result = nil
-        self.load
-      else
-        @result
-      end
+  def fork
+    Process.fork do
+      clear_info unless present?
+      run(false)
+      join
     end
+    grace
+    self
   end
 
   def done?
@@ -185,16 +199,21 @@ class Step
         else
           Log.debug "Taking result #{Log.fingerprint @result}"
         end
+
         @take_stream, @result = @result, @result.next
-        @take_stream
-      elsif done?
+
+        return @take_stream
+      end
+    end
+
+    if done?
+      Open.open(self.path)
+    else
+      if running?
+        join
         Open.open(self.path)
       else
-        if running?
-          nil
-        else
-          exec
-        end
+        exec
       end
     end
   end
@@ -207,21 +226,47 @@ class Step
     threads.each do |t| 
       begin
         t.join 
-      rescue
+      rescue Exception
         threads.each{|t| t.raise(Aborted); t.join }
         raise $!
       end
     end
   end
 
+  def present?
+    Open.exist?(path) ||
+      Open.exist?(info_file) ||
+      Open.exist?(files_dir)
+  end
+
+  def grace
+    while ! present?
+      sleep 0.1
+    end
+  end
+
+  def terminated?
+    ! @in_exec && (done? || error? || aborted?)
+  end
+
   def join
     consume_all_streams
+    while @result.nil? && (present? && ! terminated?)
+      sleep 0.1
+    end
+    raise self.exception if self.exception
+    raise "Error in job #{self.path}" if self.error? or self.aborted? 
     self
   end
 
-  def produce
-    run
-    join
+  def produce(with_fork: false)
+    if with_fork
+      self.fork
+      self.join
+    else
+      run
+    end
+    self
   end
 
   def load

data/lib/scout/workflow/task/dependencies.rb

@@ -0,0 +1,114 @@
+module Task
+  def dependencies(id, provided_inputs, non_default_inputs = [], compute = {})
+    return [] if deps.nil?
+    dependencies = []
+    
+    provided_inputs ||= {}
+
+    # Helper function
+    load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
+      task = step_options.delete(:task) if step_options.include?(:task)
+      workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
+      id = step_options.delete(:id) if step_options.include?(:id)
+      id = step_options.delete(:jobname) if step_options.include?(:jobname)
+
+      step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
+      step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
+
+      resolved_inputs = {}
+      step_inputs.each do |k,v|
+        if Symbol === v
+          input_dep = dependencies.select{|d| d.task_name == v }.first
+          resolved_inputs[k] = input_dep || step_inputs[v] || k
+        else
+          resolved_inputs[k] = v
+        end
+      end
+      job = workflow.job(task, id, resolved_inputs)
+      compute_options = definition_options[:compute] || []
+      compute_options = [compute_options] unless Array === compute_options
+      compute_options << :canfail if definition_options[:canfail]
+      compute_options << :produce if definition_options[:produce]
+      compute_options << :stream if definition_options[:stream]
+      compute[job.path] = compute_options if compute_options.any?
+      [job, step_inputs]
+    end
+
+    # Helper function
+    find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
+      dep_non_default_inputs = dep.non_default_inputs
+      dep_non_default_inputs.select do |name|
+        step_inputs.include?(name)  
+      end
+      dep_non_default_inputs.reject! do |name|
+        definition_options.include?(name) && 
+          (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
+      end
+
+      dep_non_default_inputs
+    end
+
+    deps.each do |workflow,task,definition_options,block=nil|
+      definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
+
+      if provided_inputs.include?(overriden = [workflow.name, task] * "#")
+        dep = provided_inputs[overriden]
+        dep = Step.new dep unless Step === dep
+        dep.type = workflow.tasks[task].type
+        dependencies << dep
+        non_default_inputs << overriden
+        next
+      end
+
+      definition_options ||= {}
+
+      if block
+        fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
+        self.inputs.each do |name,type,desc,value|
+          fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
+        end
+        fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
+        block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
+
+        res = block.call id, block_options, dependencies
+
+        case res
+        when Step
+          dep = res
+          dependencies << dep
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Hash
+          step_options = block_options.merge(res)
+          dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+          dependencies << dep
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Array
+          res.each do |_res|
+            if Hash === _res
+              step_options = block_options.merge(_res)
+              dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+              dependencies << dep
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+              non_default_inputs.concat(dep_non_default_inputs)
+            else
+              dep = _res
+              dependencies << dep
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+              non_default_inputs.concat(dep_non_default_inputs)
+            end
+          end
+        end
+      else
+        step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
+        dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
+        dependencies << dep
+        dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+        non_default_inputs.concat(dep_non_default_inputs)
+      end
+    end
+
+    dependencies
+  end
+end

data/lib/scout/workflow/task/inputs.rb

@@ -3,7 +3,7 @@ module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value
 
-    if String === value && ! [:path, :file, :folder].include?(type) && (options.nil? || (! options[:noload] || options[:stream]))
+    if String === value && ! [:path, :file, :folder, :binary].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
       if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else
@@ -63,7 +63,7 @@ module Task
     basename = File.basename(orig_file)
     digest = Misc.digest(orig_file)
     if basename.include? '.'
-      basename.sub!(/(.*)\.(.*)/, "\1-#{digest}.\2")
+      basename.sub!(/(.*)\.(.*)/, '\1-' + digest + '.\2')
     else
       basename += "-#{digest}"
     end
@@ -74,35 +74,49 @@ module Task
   end
 
   def save_inputs(directory, provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs(provided_inputs)
-    self.inputs.each_with_index do |p,i|
+    #input_array, non_default_inputs = assign_inputs(provided_inputs)
+    self.recursive_inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
-      next unless non_default_inputs.include?(name)
+      next unless provided_inputs.include?(name)
+      value = provided_inputs[name]
       input_file = File.join(directory, name.to_s)
 
       if type == :file
-        relative_file = save_file_input(input_array[i], directory)
-        Persist.save(relative_file, input_file, type)
+        relative_file = save_file_input(value, directory)
+        Persist.save(relative_file, input_file, :file)
       elsif type == :file_array
-        new_files = input_array[i].collect do |orig_file|
+        new_files = value.collect do |orig_file|
           save_file_input(orig_file, directory)
         end
         Persist.save(new_files, input_file, type)
+      elsif Path.is_filename?(value) 
+        relative_file = save_file_input(value, directory)
+        Open.write(input_file + ".as_file", relative_file)
+      elsif Open.is_stream?(value)
+        Persist.save(input_file, value, type)
+      elsif Open.has_stream?(value)
+        Persist.save(input_file, value.stream, type)
       else
-        Persist.save(input_array[i], input_file, type)
+        Persist.save(value, input_file, type)
       end
     end
   end
 
   def load_inputs(directory)
-    self.inputs.collect do |p|
+    inputs = IndiferentHash.setup({})
+    self.recursive_inputs.each do |p|
       name, type, desc, value, options = p
       filename = File.join(directory, name.to_s) 
       if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
-        Persist.load(filename, type)
-      else
-        value
+        if filename.end_with?('.as_file')
+          value = Open.read(filename).strip
+          value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
+          inputs[name] = value
+        else
+          inputs[name] = Persist.load(filename, type)
+        end
       end
     end
+    inputs
   end
 end

data/lib/scout/workflow/task.rb

@@ -2,6 +2,7 @@ require_relative '../meta_extension'
 require_relative '../named_array'
 require_relative 'step'
 require_relative 'task/inputs'
+require_relative 'task/dependencies'
 
 module Task
   extend MetaExtension
@@ -38,123 +39,22 @@ module Task
     binding.instance_exec(*inputs, &self)
   end
 
-  def dependencies(id, provided_inputs, non_default_inputs = [])
-    return [] if deps.nil?
-    dependencies = []
-    
-    provided_inputs ||= {}
-
-    # Helper function
-    load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
-      task = step_options.delete(:task) if step_options.include?(:task)
-      workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
-      id = step_options.delete(:id) if step_options.include?(:id)
-      id = step_options.delete(:jobname) if step_options.include?(:jobname)
-
-      step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
-      step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
-
-      resolved_inputs = {}
-      step_inputs.each do |k,v|
-        if Symbol === v
-          input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || step_inputs[v] || k
-        else
-          resolved_inputs[k] = v
-        end
-      end
-      [workflow.job(task, id, resolved_inputs), step_inputs]
-    end
-
-    # Helper function
-    find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
-      dep_non_default_inputs = dep.non_default_inputs
-      dep_non_default_inputs.select do |name|
-        step_inputs.include?(name)  
-      end
-      dep_non_default_inputs.reject! do |name|
-        definition_options.include?(name) && 
-          (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
-      end
-
-      dep_non_default_inputs
-    end
-
-    deps.each do |workflow,task,definition_options,block=nil|
-      definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
-
-      if provided_inputs.include?(overriden = [workflow.name, task] * "#")
-        dep = provided_inputs[overriden]
-        dep = Step.new dep unless Step === dep
-        dep.type = workflow.tasks[task].type
-        dependencies << dep
-        non_default_inputs << overriden
-        next
-      end
-
-      definition_options ||= {}
-
-      if block
-        fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
-        self.inputs.each do |name,type,desc,value|
-          fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
-        end
-        fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
-        block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
-
-        res = block.call id, block_options, dependencies
-
-        case res
-        when Step
-          dep = res
-          dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
-          non_default_inputs.concat(dep_non_default_inputs)
-        when Hash
-          step_options = block_options.merge(res)
-          dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
-          dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
-          non_default_inputs.concat(dep_non_default_inputs)
-        when Array
-          res.each do |_res|
-            if Hash === _res
-              step_options = block_options.merge(_res)
-              dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
-              dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
-              non_default_inputs.concat(dep_non_default_inputs)
-            else
-              dep = _res
-              dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
-              non_default_inputs.concat(dep_non_default_inputs)
-            end
-          end
-        end
-      else
-        step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
-        dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
-        dependencies << dep
-        dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
-        non_default_inputs.concat(dep_non_default_inputs)
-      end
-    end
-
-    dependencies
-  end
-
   def job(id = DEFAULT_NAME, provided_inputs = nil )
     provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
     provided_inputs = {} if provided_inputs.nil?
     id = DEFAULT_NAME if id.nil?
 
+    provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
+
     inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
 
-    dependencies = dependencies(id, provided_inputs, non_default_inputs)
+    compute = {}
+    dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
     non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
+    non_default_inputs.uniq!
+
     if non_default_inputs.any?
       hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
       name = [id, hash] * "_"
@@ -178,6 +78,7 @@ module Task
       end
     end
 
+
     path = directory[name]
 
     path = path.set_extension(extension) if extension
@@ -189,7 +90,7 @@ module Task
         Log.debug "ID #{self.name} #{id} - Clean"
       end
       NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
-      Step.new path.find, inputs, dependencies, id, non_default_inputs, &self
+      Step.new path.find, inputs, dependencies, id, non_default_inputs, compute, &self
     end
   end
 end

data/lib/scout/workflow/usage.rb

@@ -3,10 +3,31 @@ require 'scout/simple_opt'
 module Task
   def usage(workflow = nil, deps = nil)
     str = StringIO.new
-    str.puts Log.color(:yellow, name)
-    str.puts Log.color(:yellow, "-" * name.length)
-    str.puts "\n" << Misc.format_paragraph(description.strip)  << "\n" if description and not description.empty?
-    str.puts
+
+    if description
+      title, paragraph = description.split("\n\n")
+      if title.length < Misc::MAX_TTY_LINE_WIDTH
+        title = self.name.to_s + " - " + title
+        str.puts Log.color :yellow, title
+        str.puts Log.color :yellow, "-" * title.length
+        if paragraph
+          str.puts "\n" << Misc.format_paragraph(paragraph) 
+        end
+        str.puts
+      else
+        title = self.name.to_s
+        str.puts Log.color :yellow, title
+        str.puts Log.color :yellow, "-" * title.length
+        str.puts "\n" << Misc.format_paragraph(description) 
+        str.puts
+      end
+    else
+      title = self.name.to_s
+      str.puts Log.color :yellow, title
+      str.puts Log.color :yellow, "-" * title.length
+      str.puts
+    end
+
 
     selects = []
     if inputs && inputs.any?
@@ -103,9 +124,16 @@ module Task
     sopt_options * ":"
   end
 
-  def get_SOPT(task)
+  def get_SOPT
     sopt_option_string = self.SOPT_str
-    SOPT.get sopt_option_string
+    job_options = SOPT.get sopt_option_string
+    recursive_inputs.uniq.each do |name,type|
+      next unless type.to_s.include?('array')
+      if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
+        job_options[name] = job_options[name].split(",")
+      end
+    end
+    job_options
   end
 end
 
@@ -219,15 +247,15 @@ module Workflow
       str.puts Log.color :magenta, "=" * self.name.length
     end
 
-    if self.documentation[:description] and not self.documentation[:description].empty?
-      str.puts
-      str.puts Misc.format_paragraph self.documentation[:description] 
-      str.puts
-    end
-
+    str.puts
 
     if task.nil?
 
+      if self.documentation[:description] and not self.documentation[:description].empty?
+        str.puts Misc.format_paragraph self.documentation[:description] 
+        str.puts
+      end
+
       str.puts Log.color :magenta, "## TASKS"
       if self.documentation[:task_description] and not self.documentation[:task_description].empty?
         str.puts

data/lib/scout/workflow.rb

@@ -4,13 +4,14 @@ require_relative 'workflow/task'
 require_relative 'workflow/step'
 require_relative 'workflow/documentation'
 require_relative 'workflow/usage'
+require_relative 'workflow/deployment'
 
 require_relative 'resource'
 require_relative 'resource/scout'
 
 module Workflow
   class << self
-    attr_accessor :workflows
+    attr_accessor :workflows, :main
     def workflows
       @workflows ||= []
     end
@@ -30,13 +31,14 @@ module Workflow
     workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
     workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
     if Open.exists?(workflow)
+      self.main = nil
       workflow = workflow.find if Path === workflow
       $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
       load workflow
     else
       raise "Workflow #{workflow_name} not found"
     end
-    workflows.last
+    self.main || workflows.last
   end
 
   def job(name, *args)

data/lib/scout-gear.rb

@@ -8,3 +8,5 @@ require_relative 'scout/resource'
 require_relative 'scout/resource/scout'
 require_relative 'scout/persist'
 require_relative 'scout/tsv'
+require_relative 'scout/config'
+require_relative 'scout/offsite'

data/lib/scout.rb

@@ -1,2 +1,8 @@
 require 'workflow-scout'
 require 'rbbt-scout'
+
+module Scout
+  def self.version
+    Open.read(File.join(__dir__, '../VERSION'))
+  end
+end