scout-gear 2.0.0 → 5.1.1

data/lib/scout/workflow/step/info.rb

@@ -0,0 +1,77 @@
+class Step
+  def info_file
+    @info_file ||= @path + ".info"
+  end
+
+  def load_info
+    @info = Persist.load(info_file, :marshal) || {}
+    @info_load_time = Time.now
+  end
+
+  def save_info
+    Persist.save(@info, info_file, :marshal)
+    @info_load_time = Time.now
+  end
+
+  def info
+    outdated = @info && Open.exists?(info_file) && @info_load_time && Open.mtime(info_file) > @info_load_time
+
+    if @info.nil? || outdated
+      load_info
+    end
+
+    @info
+  end
+
+  def merge_info(new_info)
+    info = self.info
+    new_info.each do |key,value|
+      report_status new_info[:status], new_info[:message] if key == :status
+      if info.include?(key)
+        case info[key]
+        when Array
+          info[key].concat Array === value ? value : [value]
+        when Hash
+          info[key].merge! value
+        else
+          info[key] = value
+        end
+      else
+        info[key] = value
+      end
+    end
+    save_info
+  end
+
+  def set_info(key, value)
+    merge_info(key => value)
+  end
+  
+  def init_info
+    @info = {
+      :status => :waiting
+    }
+  end
+
+  def report_status(status, message = nil)
+    if message.nil?
+      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)
+    else
+      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path) + " " + message
+    end
+  end
+
+  def log(status, message = nil)
+    report_status status, message
+    if message
+      merge_info :status => status, :messages => [message]
+    else
+      merge_info :status => status
+    end
+  end
+
+  def status
+    info[:status]
+  end
+
+end

data/lib/scout/workflow/step.rb

@@ -0,0 +1,96 @@
+require_relative '../path'
+require_relative '../persist'
+require_relative 'step/info'
+
+class Step 
+
+  attr_accessor :path, :inputs, :dependencies, :task
+  def initialize(path, inputs = nil, dependencies = [], &task) 
+    @path = path
+    @inputs = inputs
+    @dependencies = dependencies
+    @task = task
+  end
+
+  attr_accessor :type
+  def type
+    @type ||= @task.respond_to?(:type) ? @task.type : nil
+  end
+
+  def name
+    @name ||= File.basename(@path)
+  end
+
+  def task_name
+    @task_name ||= @task.name if @task.respond_to?(:name)
+  end
+
+  def exec
+    self.instance_exec(*inputs, &task)
+  end
+
+  attr_reader :result
+  def run
+    @result = Persist.persist(name, type, :path => path) do
+      begin
+        merge_info :status => :start, :start => Time.now,
+          :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
+          :inputs => inputs,
+          :dependencies => dependencies.collect{|d| d.path }
+
+        dependencies.each{|dep| dep.run }
+        @result = exec
+      ensure
+        if streaming?
+          ConcurrentStream.setup(@result) do
+            merge_info :status => :done, :end => Time.now
+          end
+          log :streaming
+        else
+          merge_info :status => :done, :end => Time.now
+        end
+      end
+    end
+  end
+
+  def done?
+    Open.exist?(path)
+  end
+
+  def streaming?
+    IO === @result || StringIO === @result
+  end
+
+  def join
+    if streaming?
+      Open.consume_stream(@result, false) 
+      @result = nil
+    end
+  end
+
+  def produce
+    run
+    join
+  end
+
+  def load
+    return @result unless @result.nil? || streaming?
+    join
+    done? ? Persist.load(path, type) : exec
+  end
+
+  def clean
+    FileUtils.rm path.find if path.exist?
+  end
+
+  def step(task_name)
+    dependencies.each do |dep|
+      return dep if dep.task_name == task_name
+    end
+    nil
+  end
+
+  def digest_str
+    path
+  end
+end

data/lib/scout/workflow/task/inputs.rb

@@ -0,0 +1,112 @@
+module Task
+
+  def format_input(value, type, options = {})
+    return value if IO === value || StringIO === value
+
+    value = value.load if Step === value
+    if String === value && ! [:path, :file].include?(type) 
+      if Open.exists?(value)
+        Persist.load(value, type) 
+      else
+        Persist.deserialize(value, type)
+      end
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        if Array === value
+          value.collect{|v| format_input(v, m[1].to_sym, options) }
+        end
+      else
+        value
+      end
+    end
+  end
+
+  def assign_inputs(provided_inputs = {})
+    if self.inputs.nil?
+      case provided_inputs
+      when Array
+        return [provided_inputs, provided_inputs]
+      else
+        return [[], []]
+      end
+    end
+
+    input_array = []
+    non_default_inputs = []
+    self.inputs.each_with_index do |p,i|
+      name, type, desc, value, options = p
+      provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
+      provided = format_input(provided, type, options || {})
+      if ! provided.nil? && provided != value
+        non_default_inputs << name
+        input_array << provided
+      else
+        input_array << value
+      end
+    end
+
+    [input_array, non_default_inputs]
+  end
+
+  def digest_inputs(provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    if Array === provided_inputs 
+      Misc.digest(input_array)
+    else
+      Misc.digest(input_array)
+    end
+  end
+  
+  def process_inputs(provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    digest = Misc.digest(input_array)
+    [input_array, non_default_inputs, digest]
+  end
+
+  def save_file_input(orig_file, directory)
+    basename = File.basename(orig_file)
+    digest = Misc.digest(orig_file)
+    if basename.include? '.'
+      basename.sub!(/(.*)\.(.*)/, "\1-#{digest}.\2")
+    else
+      basename += "-#{digest}"
+    end
+    new_file = File.join(directory, 'saved_input_files', basename)
+    relative_file = File.join('.', 'saved_input_files', basename) 
+    Open.link orig_file, new_file
+    relative_file
+  end
+
+  def save_inputs(directory, provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    self.inputs.each_with_index do |p,i|
+      name, type, desc, value, options = p
+      next unless non_default_inputs.include?(name)
+      input_file = File.join(directory, name.to_s)
+
+      if type == :file
+        relative_file = save_file_input(input_array[i], directory)
+        Persist.save(relative_file, input_file, type)
+      elsif type == :file_array
+        new_files = input_array[i].collect do |orig_file|
+          save_file_input(orig_file, directory)
+        end
+        Persist.save(new_files, input_file, type)
+      else
+        Persist.save(input_array[i], input_file, type)
+      end
+    end
+  end
+
+  def load_inputs(directory)
+    self.inputs.collect do |p|
+      name, type, desc, value, options = p
+      filename = File.join(directory, name.to_s) 
+      if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
+        Persist.load(filename, type)
+      else
+        value
+      end
+    end
+  end
+end

data/lib/scout/workflow/task.rb

@@ -0,0 +1,141 @@
+require_relative '../meta_extension'
+require_relative 'step'
+require_relative 'task/inputs'
+
+module Task
+  extend MetaExtension
+  extension_attr :name, :type, :inputs, :deps, :directory, :description
+
+  DEFAULT_NAME = "Default"
+
+  class << self
+    attr_accessor :default_directory
+
+    def default_directory
+      @default_directory ||= Path.setup('var/jobs/Task')
+    end
+  end
+
+  def directory
+    @directory ||= Task.default_directory
+  end
+
+  def exec_on(binding = self, *inputs)
+    binding.instance_exec(*inputs, &self)
+  end
+
+  def dependencies(id, provided_inputs, non_default_inputs = [])
+    return [] if deps.nil?
+    dependencies = []
+    
+    provided_inputs ||= {}
+
+    load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
+      task = hash_options[:task] if hash_options.include?(:task)
+      workflow = hash_options[:workflow] if hash_options.include?(:workflow)
+      id = hash_options[:id] if hash_options.include? :id
+
+      hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
+      inputs = IndiferentHash.add_defaults hash_inputs, inputs
+
+      resolved_inputs = {}
+      inputs.each do |k,v|
+        if Symbol === v
+          input_dep = dependencies.select{|d| d.task_name == v}.first
+          resolved_inputs[k] = input_dep || inputs[v] || k
+        else
+          resolved_inputs[k] = v
+        end
+      end
+      workflow.job(task, id, resolved_inputs)
+    end
+
+    deps.each do |workflow,task,options,block=nil|
+      if provided_inputs.include?(overriden = [workflow.name, task] * "#")
+        dep = provided_inputs[overriden]
+        dep = Step.new dep unless Step === dep
+        dep.type = workflow.tasks[task].type
+        dependencies << dep
+        non_default_inputs << overriden
+        next
+      end
+
+      options ||= {}
+      if block
+        inputs = IndiferentHash.add_defaults options.dup, provided_inputs
+
+        res = block.call id, inputs, dependencies
+
+        case res
+        when Step
+          dep = res
+          dependencies << dep
+          dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+          non_default_inputs.concat(dep_non_default_inputs - options.keys)
+        when Hash
+          new_options = res
+          dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
+          dependencies << dep
+          dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+          dep_non_default_inputs -= options.keys 
+          if new_options.include?(:inputs)
+            dep_non_default_inputs -= new_options[:inputs].keys 
+          else
+            dep_non_default_inputs -= new_options.keys
+          end
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Array
+          res.each do |_res|
+            if Hash === _res
+              new_options = _res
+              dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
+              dependencies << dep
+              dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+              dep_non_default_inputs -= options.keys 
+              if new_options.include?(:inputs)
+                dep_non_default_inputs -= new_options[:inputs].keys 
+              else
+                dep_non_default_inputs -= new_options.keys
+              end
+              non_default_inputs.concat(dep_non_default_inputs)
+            else
+              dep = _res
+              dependencies << dep
+              dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+              non_default_inputs.concat(dep_non_default_inputs - options.keys)
+            end
+          end
+        end
+      else
+        inputs = IndiferentHash.add_defaults options.dup, provided_inputs
+        dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
+        dependencies << dep
+        dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+        non_default_inputs.concat(dep_non_default_inputs - options.keys)
+      end
+    end
+
+    dependencies
+  end
+
+  def job(id = DEFAULT_NAME, provided_inputs = nil )
+    provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
+    provided_inputs = {} if provided_inputs.nil?
+
+    inputs, non_default_inputs, input_hash = process_inputs provided_inputs
+
+    dependencies = dependencies(id, provided_inputs, non_default_inputs)
+
+    non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
+
+    if non_default_inputs.any?
+      hash = Misc.digest(:inputs => input_hash, :non_default_inputs => non_default_inputs, :dependencies => dependencies)
+      Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :dependencies => dependencies)}"
+      id = [id, hash] * "_"
+    end
+
+    path = directory[id]
+
+    Step.new path, inputs, dependencies, &self
+  end
+end