scout-gear 2.0.0 → 5.2.0

data/lib/scout/workflow/documentation.rb

@@ -0,0 +1,83 @@
+module Workflow
+  attr_accessor :title, :description
+
+  def self.doc_parse_first_line(str)
+    if str.match(/^([^\n]*)\n\n(.*)/sm)
+      str.replace $2
+      $1
+    else
+      ""
+    end
+  end
+
+  def self.doc_parse_up_to(str, pattern, keep = false)
+    pre, _pat, _post = str.partition pattern
+    if _pat
+      [pre, (keep ? _pat << _post : _post)]
+    else
+      _post
+    end
+  end
+
+  def self.doc_parse_chunks(str, pattern)
+    parts = str.split(pattern)
+    return {} if parts.length < 2
+    tasks = Hash[*parts[1..-1].collect{|v| v.strip}]
+    tasks.delete_if{|t,d| d.empty?}
+    tasks
+  end
+
+  def self.parse_workflow_doc(doc)
+    title = doc_parse_first_line doc
+    description, task_info = doc_parse_up_to doc, /^# Tasks/i
+    task_description, tasks = doc_parse_up_to task_info, /^##/, true
+    tasks = doc_parse_chunks tasks, /## (.*)/ 
+    {:title => title.strip, :description => description.strip, :task_description => task_description.strip, :tasks => tasks}
+  end
+
+  def documentation_markdown
+    return "" if @libdir.nil?
+    file = @libdir['workflow.md'].find
+    file = @libdir['README.md'].find unless file.exists?
+    if file.exists?
+      file.read
+    else
+      ""
+    end
+  end
+
+  attr_accessor :documentation
+  def documentation
+    @documentation ||= begin
+                         documentation = Workflow.parse_workflow_doc documentation_markdown
+
+                         if @description && (documentation[:description].nil? || documentation[:description].empty?)
+                           documentation[:description] = @description 
+                         end
+
+                         if @title && (documentation[:title].nil? || documentation[:title].empty?)
+                           documentation[:title] = @title 
+                         end
+                         documentation[:tasks].each do |task, description|
+                           if task.include? "#"
+                             workflow, task = task.split("#")
+                             workflow = begin
+                                          Kernel.const_get workflow
+                                        rescue
+                                          next
+                                        end
+                           else
+                             workflow = self
+                           end
+
+                           task = task.to_sym
+                           if workflow.tasks.include? task
+                             workflow.tasks[task].description = description
+                           else
+                             Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
+                           end
+                         end
+                         documentation
+                       end
+  end
+end

data/lib/scout/workflow/step/info.rb

@@ -0,0 +1,77 @@
+class Step
+  SERIALIZER = :json
+  def info_file
+    @info_file ||= begin
+                     info_file = @path + ".info"
+                     @path.annotate info_file if Path === @path
+                     info_file
+                   end
+  end
+
+  def load_info
+    @info = Persist.load(info_file, SERIALIZER) || {}
+    IndiferentHash.setup(@info)
+    @info_load_time = Time.now
+  end
+
+  def save_info(info = nil)
+    Persist.save(info, info_file, SERIALIZER)
+    @info_load_time = Time.now
+  end
+
+  def info
+    outdated = @info && Open.exists?(info_file) && @info_load_time && Open.mtime(info_file) > @info_load_time
+
+    if @info.nil? || outdated
+      load_info
+    end
+
+    @info
+  end
+
+  def merge_info(new_info)
+    info = self.info
+    new_info.each do |key,value|
+      report_status new_info[:status], new_info[:message] if key == :status
+      if info.include?(key)
+        case info[key]
+        when Array
+          info[key].concat Array === value ? value : [value]
+        when Hash
+          info[key].merge! value
+        else
+          info[key] = value
+        end
+      else
+        info[key] = value
+      end
+    end
+    save_info(info)
+  end
+
+  def set_info(key, value)
+    merge_info(key => value)
+  end
+  
+  def report_status(status, message = nil)
+    if message.nil?
+      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)
+    else
+      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path) + " " + message
+    end
+  end
+
+  def log(status, message = nil)
+    report_status status, message
+    if message
+      merge_info :status => status, :messages => [message]
+    else
+      merge_info :status => status
+    end
+  end
+
+  def status
+    info[:status]
+  end
+
+end

data/lib/scout/workflow/step/load.rb

@@ -0,0 +1,18 @@
+class Step
+  def self.relocate(path)
+    return path if Open.exists?(path)
+    Path.setup(path) unless Path === path
+    relocated = path.relocate
+    return relocated if Open.exists?(relocated)
+    subpath = path.split("/")[-3..-1] * "/"
+    relocated = Path.setup("var/jobs")[subpath]
+    return relocated if Open.exists?(relocated)
+    path
+  end
+
+  def self.load(path)
+    path = relocate(path) unless Open.exists?(path)
+    raise "Could not load #{path}" unless Open.exists?(path)
+    s = Step.new path
+  end
+end

data/lib/scout/workflow/step.rb

@@ -0,0 +1,132 @@
+require_relative '../path'
+require_relative '../persist'
+require_relative 'step/info'
+require_relative 'step/load'
+
+class Step 
+
+  attr_accessor :path, :inputs, :dependencies, :task
+  def initialize(path, inputs = nil, dependencies = nil, &task) 
+    @path = path
+    @inputs = inputs
+    @dependencies = dependencies
+    @task = task
+  end
+
+  def inputs
+    @inputs ||= begin
+                  if Open.exists?(info_file)
+                    info[:inputs]
+                  else
+                    []
+                  end
+                end
+  end
+
+  def dependencies
+    @dependencies ||= begin
+                        if Open.exists?(info_file)
+                          info[:dependencies].collect do |path|
+                            Step.load(path)
+                          end
+                        else
+                          []
+                        end
+                      end
+  end
+
+  attr_accessor :type
+  def type
+    @type ||= @task.respond_to?(:type) ? @task.type : info[:type]
+  end
+
+  def name
+    @name ||= File.basename(@path)
+  end
+
+  def task_name
+    @task_name ||= @task.name if @task.respond_to?(:name)
+  end
+
+  def exec
+    self.instance_exec(*inputs, &task)
+  end
+
+  attr_reader :result
+  def run
+    return @result || self.load if done?
+    dependencies.each{|dep| dep.run }
+    @result = Persist.persist(name, type, :path => path) do
+      begin
+        merge_info :status => :start, :start => Time.now,
+          :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
+          :inputs => inputs, :type => type,
+          :dependencies => dependencies.collect{|d| d.path }
+
+        @result = exec
+      ensure
+        if streaming?
+          ConcurrentStream.setup(@result) do
+            merge_info :status => :done, :end => Time.now
+          end
+          log :streaming
+        else
+          merge_info :status => :done, :end => Time.now
+        end
+      end
+    end
+  end
+
+  def done?
+    Open.exist?(path)
+  end
+
+  def streaming?
+    IO === @result || StringIO === @result
+  end
+
+  def stream
+    join
+    streaming? ? @result : Open.open(path)
+  end
+
+  def join
+    if streaming?
+      Open.consume_stream(@result, false) 
+      @result = nil
+    end
+  end
+
+  def produce
+    run
+    join
+  end
+
+  def load
+    return @result unless @result.nil? || streaming?
+    join
+    done? ? Persist.load(path, type) : exec
+  end
+
+  def clean
+    FileUtils.rm path.find if path.exist?
+  end
+
+  def recursive_clean
+    dependencies.each do |dep|
+      dep.recursive_clean
+    end
+    clean
+  end
+
+  def step(task_name)
+    dependencies.each do |dep|
+      return dep if dep.task_name == task_name
+    end
+    nil
+  end
+
+  def digest_str
+    path
+  end
+end

data/lib/scout/workflow/task/inputs.rb

@@ -0,0 +1,114 @@
+module Task
+
+  def format_input(value, type, options = {})
+    return value if IO === value || StringIO === value
+
+    value = value.load if Step === value
+    if String === value && ! [:path, :file].include?(type) 
+      if Open.exists?(value)
+        Persist.load(value, type) 
+      else
+        Persist.deserialize(value, type)
+      end
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        if Array === value
+          value.collect{|v| format_input(v, m[1].to_sym, options) }
+        end
+      else
+        value
+      end
+    end
+  end
+
+  def assign_inputs(provided_inputs = {})
+    if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
+      case provided_inputs
+      when Array
+        return [provided_inputs, provided_inputs]
+      else
+        return [[], []]
+      end
+    end
+
+    IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
+
+    input_array = []
+    non_default_inputs = []
+    self.inputs.each_with_index do |p,i|
+      name, type, desc, value, options = p
+      provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
+      provided = format_input(provided, type, options || {})
+      if ! provided.nil? && provided != value
+        non_default_inputs << name
+        input_array << provided
+      else
+        input_array << value
+      end
+    end
+
+    [input_array, non_default_inputs]
+  end
+
+  def digest_inputs(provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    if Array === provided_inputs 
+      Misc.digest(input_array)
+    else
+      Misc.digest(input_array)
+    end
+  end
+  
+  def process_inputs(provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs provided_inputs
+    digest = Misc.digest(input_array)
+    [input_array, non_default_inputs, digest]
+  end
+
+  def save_file_input(orig_file, directory)
+    basename = File.basename(orig_file)
+    digest = Misc.digest(orig_file)
+    if basename.include? '.'
+      basename.sub!(/(.*)\.(.*)/, "\1-#{digest}.\2")
+    else
+      basename += "-#{digest}"
+    end
+    new_file = File.join(directory, 'saved_input_files', basename)
+    relative_file = File.join('.', 'saved_input_files', basename) 
+    Open.link orig_file, new_file
+    relative_file
+  end
+
+  def save_inputs(directory, provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    self.inputs.each_with_index do |p,i|
+      name, type, desc, value, options = p
+      next unless non_default_inputs.include?(name)
+      input_file = File.join(directory, name.to_s)
+
+      if type == :file
+        relative_file = save_file_input(input_array[i], directory)
+        Persist.save(relative_file, input_file, type)
+      elsif type == :file_array
+        new_files = input_array[i].collect do |orig_file|
+          save_file_input(orig_file, directory)
+        end
+        Persist.save(new_files, input_file, type)
+      else
+        Persist.save(input_array[i], input_file, type)
+      end
+    end
+  end
+
+  def load_inputs(directory)
+    self.inputs.collect do |p|
+      name, type, desc, value, options = p
+      filename = File.join(directory, name.to_s) 
+      if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
+        Persist.load(filename, type)
+      else
+        value
+      end
+    end
+  end
+end

data/lib/scout/workflow/task.rb

@@ -0,0 +1,155 @@
+require_relative '../meta_extension'
+require_relative 'step'
+require_relative 'task/inputs'
+
+module Task
+  extend MetaExtension
+  extension_attr :name, :type, :inputs, :deps, :directory, :description
+
+  DEFAULT_NAME = "Default"
+
+  class << self
+    attr_accessor :default_directory
+
+    def default_directory
+      @default_directory ||= Path.setup('var/jobs/Task')
+    end
+  end
+
+  def inputs
+    @inputs ||= []
+  end
+
+  def recursive_inputs
+    return inputs if deps.nil?
+    deps.inject(inputs) do |acc,dep|
+      workflow, task = dep
+      next if workflow.nil?
+      acc += workflow.tasks[task].recursive_inputs
+    end
+  end
+
+  def directory
+    @directory ||= Task.default_directory
+  end
+
+  def exec_on(binding = self, *inputs)
+    binding.instance_exec(*inputs, &self)
+  end
+
+  def dependencies(id, provided_inputs, non_default_inputs = [])
+    return [] if deps.nil?
+    dependencies = []
+    
+    provided_inputs ||= {}
+
+    load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
+      task = hash_options[:task] if hash_options.include?(:task)
+      workflow = hash_options[:workflow] if hash_options.include?(:workflow)
+      id = hash_options[:id] if hash_options.include? :id
+
+      hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
+      inputs = IndiferentHash.add_defaults hash_inputs, inputs
+
+      resolved_inputs = {}
+      inputs.each do |k,v|
+        if Symbol === v
+          input_dep = dependencies.select{|d| d.task_name == v}.first
+          resolved_inputs[k] = input_dep || inputs[v] || k
+        else
+          resolved_inputs[k] = v
+        end
+      end
+      workflow.job(task, id, resolved_inputs)
+    end
+
+    deps.each do |workflow,task,options,block=nil|
+      if provided_inputs.include?(overriden = [workflow.name, task] * "#")
+        dep = provided_inputs[overriden]
+        dep = Step.new dep unless Step === dep
+        dep.type = workflow.tasks[task].type
+        dependencies << dep
+        non_default_inputs << overriden
+        next
+      end
+
+      options ||= {}
+      if block
+        inputs = IndiferentHash.add_defaults options.dup, provided_inputs
+
+        res = block.call id, inputs, dependencies
+
+        case res
+        when Step
+          dep = res
+          dependencies << dep
+          dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+          non_default_inputs.concat(dep_non_default_inputs - options.keys)
+        when Hash
+          new_options = res
+          dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
+          dependencies << dep
+          dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+          dep_non_default_inputs -= options.keys 
+          if new_options.include?(:inputs)
+            dep_non_default_inputs -= new_options[:inputs].keys 
+          else
+            dep_non_default_inputs -= new_options.keys
+          end
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Array
+          res.each do |_res|
+            if Hash === _res
+              new_options = _res
+              dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
+              dependencies << dep
+              dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+              dep_non_default_inputs -= options.keys 
+              if new_options.include?(:inputs)
+                dep_non_default_inputs -= new_options[:inputs].keys 
+              else
+                dep_non_default_inputs -= new_options.keys
+              end
+              non_default_inputs.concat(dep_non_default_inputs)
+            else
+              dep = _res
+              dependencies << dep
+              dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+              non_default_inputs.concat(dep_non_default_inputs - options.keys)
+            end
+          end
+        end
+      else
+        inputs = IndiferentHash.add_defaults options.dup, provided_inputs
+        dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
+        dependencies << dep
+        dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+        non_default_inputs.concat(dep_non_default_inputs - options.keys)
+      end
+    end
+
+    dependencies
+  end
+
+  def job(id = DEFAULT_NAME, provided_inputs = nil )
+    provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
+    provided_inputs = {} if provided_inputs.nil?
+    id = DEFAULT_NAME if id.nil?
+
+    inputs, non_default_inputs, input_hash = process_inputs provided_inputs
+
+    dependencies = dependencies(id, provided_inputs, non_default_inputs)
+
+    non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
+
+    if non_default_inputs.any?
+      hash = Misc.digest(:inputs => input_hash, :non_default_inputs => non_default_inputs, :dependencies => dependencies)
+      Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :dependencies => dependencies)}"
+      id = [id, hash] * "_"
+    end
+
+    path = directory[id]
+
+    Step.new path.find, inputs, dependencies, &self
+  end
+end