scout-gear 1.2.0 → 5.1.1

data/lib/scout/tmpfile.rb

@@ -1,7 +1,10 @@
-require 'fileutils'
 require_relative 'misc'
+require_relative 'log'
+require 'fileutils'
 
 module TmpFile
+  MAX_FILE_LENGTH = 150
+
   def self.user_tmp(subdir = nil)
     if subdir
       File.join(ENV["HOME"],"/tmp/scout", subdir)
@@ -89,4 +92,39 @@ module TmpFile
       end
     end
   end
+
+  def self.tmp_for_file(file, tmp_options = {}, other_options = {})
+    tmp_for_file = IndiferentHash.process_options tmp_options, :file
+    return tmp_for_file unless tmp_for_file.nil?
+
+    prefix = IndiferentHash.process_options tmp_options, :prefix
+
+    if prefix.nil?
+      perfile = file.to_s.gsub(/\//, '>') 
+    else
+      perfile = prefix.to_s + ":" + file.to_s.gsub(/\//, '>') 
+    end
+
+    perfile.sub!(/\.b?gz$/,'')
+
+    if other_options.include? :filters
+      other_options[:filters].each do |match,value|
+        perfile = perfile + "&F[#{match}=#{Misc.digest(value)}]"
+      end
+    end
+
+    persistence_dir = IndiferentHash.process_options(tmp_options, :dir) || TmpFile.tmpdir 
+    Path.setup(persistence_dir) unless Path === persistence_dir
+
+    filename = perfile.gsub(/\s/,'_').gsub(/\//,'>')
+    clean_options = other_options.dup
+    clean_options.delete :unnamed
+    clean_options.delete "unnamed"
+
+    filename = filename[0..MAX_FILE_LENGTH] << Misc.digest(filename[MAX_FILE_LENGTH+1..-1]) if filename.length > MAX_FILE_LENGTH + 10
+
+    filename  += ":" << Misc.digest(clean_options) unless clean_options.empty?
+
+    persistence_dir[filename]
+  end
 end

data/lib/scout/workflow/definition.rb

@@ -0,0 +1,72 @@
+require_relative '../meta_extension'
+
+module Workflow
+  extend MetaExtension
+  extension_attr :name, :tasks
+
+  class << self
+    attr_accessor :directory
+
+    def directory
+      @directory ||= Path.setup('var/jobs')
+    end
+
+  end
+
+  def name
+    @name ||= self.to_s
+  end
+
+  attr_accessor :directory
+  def directory
+    @directory ||= Workflow.directory[name]
+  end
+
+  def directory=(directory)
+    @directory = directory
+    @tasks.each{|name,d| d.directory = directory[name] } if @tasks
+  end
+
+  def annotate_next_task(type, obj)
+    @annotate_next_task ||= {}
+    @annotate_next_task[type] ||= []
+    @annotate_next_task[type] << obj
+  end
+
+  def dep(*args, &block)
+    case args.length
+    when 3
+      workflow, task, options = args
+    when 2
+      if Hash === args.last
+        task, options = args
+      else
+        workflow, task = args
+      end
+    when 1
+      task = args.first
+    end
+    workflow = self if workflow.nil?
+    options = {} if options.nil?
+    annotate_next_task :deps, [workflow, task, options, block, args]
+  end
+
+  def input(*args)
+    annotate_next_task(:inputs, args)
+  end
+
+  def task(name_and_type, &block)
+    name, type = name_and_type.collect.first
+    @tasks ||= IndiferentHash.setup({})
+    begin
+      @annotate_next_task ||= {}
+      @tasks[name] = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
+    ensure
+      @annotate_next_task = {}
+    end
+  end
+
+  def desc(description)
+    annotate_next_task(:desc, description)
+  end
+end

data/lib/scout/workflow/documentation.rb

@@ -0,0 +1,77 @@
+module Workflow
+
+  def self.doc_parse_first_line(str)
+    if str.match(/^([^\n]*)\n\n(.*)/sm)
+      str.replace $2
+      $1
+    else
+      ""
+    end
+  end
+
+  def self.doc_parse_up_to(str, pattern, keep = false)
+    pre, _pat, _post = str.partition pattern
+    if _pat
+      [pre, (keep ? _pat << _post : _post)]
+    else
+      _post
+    end
+  end
+
+  def self.doc_parse_chunks(str, pattern)
+    parts = str.split(pattern)
+    return {} if parts.length < 2
+    tasks = Hash[*parts[1..-1].collect{|v| v.strip}]
+    tasks.delete_if{|t,d| d.empty?}
+    tasks
+  end
+
+  def self.parse_workflow_doc(doc)
+    title = doc_parse_first_line doc
+    description, task_info = doc_parse_up_to doc, /^# Tasks/i
+    task_description, tasks = doc_parse_up_to task_info, /^##/, true
+    tasks = doc_parse_chunks tasks, /## (.*)/ 
+    {:title => title.strip, :description => description.strip, :task_description => task_description.strip, :tasks => tasks}
+  end
+
+  def documentation_markdown
+    return "" if @libdir.nil?
+    file = @libdir['workflow.md'].find
+    file = @libdir['README.md'].find unless file.exists?
+    if file.exists?
+      file.read
+    else
+      ""
+    end
+  end
+
+  def load_documentation
+    return if @documentation
+    @documentation ||= Workflow.parse_workflow_doc documentation_markdown
+    @documentation[:tasks].each do |task, description|
+      if task.include? "#"
+        workflow, task = task.split("#")
+        workflow = begin
+                     Kernel.const_get workflow
+                   rescue
+                     next
+                   end
+      else
+        workflow = self
+      end
+
+      task = task.to_sym
+      if workflow.tasks.include? task
+        workflow.tasks[task].description = description
+      else
+        Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
+      end
+    end
+  end
+
+  attr_accessor :documentation
+  def documentation
+    load_documentation if @documentation.nil?
+    @documentation 
+  end
+end

data/lib/scout/workflow/step/info.rb

@@ -0,0 +1,77 @@
+class Step
+  def info_file
+    @info_file ||= @path + ".info"
+  end
+
+  def load_info
+    @info = Persist.load(info_file, :marshal) || {}
+    @info_load_time = Time.now
+  end
+
+  def save_info
+    Persist.save(@info, info_file, :marshal)
+    @info_load_time = Time.now
+  end
+
+  def info
+    outdated = @info && Open.exists?(info_file) && @info_load_time && Open.mtime(info_file) > @info_load_time
+
+    if @info.nil? || outdated
+      load_info
+    end
+
+    @info
+  end
+
+  def merge_info(new_info)
+    info = self.info
+    new_info.each do |key,value|
+      report_status new_info[:status], new_info[:message] if key == :status
+      if info.include?(key)
+        case info[key]
+        when Array
+          info[key].concat Array === value ? value : [value]
+        when Hash
+          info[key].merge! value
+        else
+          info[key] = value
+        end
+      else
+        info[key] = value
+      end
+    end
+    save_info
+  end
+
+  def set_info(key, value)
+    merge_info(key => value)
+  end
+  
+  def init_info
+    @info = {
+      :status => :waiting
+    }
+  end
+
+  def report_status(status, message = nil)
+    if message.nil?
+      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)
+    else
+      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path) + " " + message
+    end
+  end
+
+  def log(status, message = nil)
+    report_status status, message
+    if message
+      merge_info :status => status, :messages => [message]
+    else
+      merge_info :status => status
+    end
+  end
+
+  def status
+    info[:status]
+  end
+
+end

data/lib/scout/workflow/step.rb

@@ -0,0 +1,96 @@
+require_relative '../path'
+require_relative '../persist'
+require_relative 'step/info'
+
+class Step 
+
+  attr_accessor :path, :inputs, :dependencies, :task
+  def initialize(path, inputs = nil, dependencies = [], &task) 
+    @path = path
+    @inputs = inputs
+    @dependencies = dependencies
+    @task = task
+  end
+
+  attr_accessor :type
+  def type
+    @type ||= @task.respond_to?(:type) ? @task.type : nil
+  end
+
+  def name
+    @name ||= File.basename(@path)
+  end
+
+  def task_name
+    @task_name ||= @task.name if @task.respond_to?(:name)
+  end
+
+  def exec
+    self.instance_exec(*inputs, &task)
+  end
+
+  attr_reader :result
+  def run
+    @result = Persist.persist(name, type, :path => path) do
+      begin
+        merge_info :status => :start, :start => Time.now,
+          :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
+          :inputs => inputs,
+          :dependencies => dependencies.collect{|d| d.path }
+
+        dependencies.each{|dep| dep.run }
+        @result = exec
+      ensure
+        if streaming?
+          ConcurrentStream.setup(@result) do
+            merge_info :status => :done, :end => Time.now
+          end
+          log :streaming
+        else
+          merge_info :status => :done, :end => Time.now
+        end
+      end
+    end
+  end
+
+  def done?
+    Open.exist?(path)
+  end
+
+  def streaming?
+    IO === @result || StringIO === @result
+  end
+
+  def join
+    if streaming?
+      Open.consume_stream(@result, false) 
+      @result = nil
+    end
+  end
+
+  def produce
+    run
+    join
+  end
+
+  def load
+    return @result unless @result.nil? || streaming?
+    join
+    done? ? Persist.load(path, type) : exec
+  end
+
+  def clean
+    FileUtils.rm path.find if path.exist?
+  end
+
+  def step(task_name)
+    dependencies.each do |dep|
+      return dep if dep.task_name == task_name
+    end
+    nil
+  end
+
+  def digest_str
+    path
+  end
+end

data/lib/scout/workflow/task/inputs.rb

@@ -0,0 +1,112 @@
+module Task
+
+  def format_input(value, type, options = {})
+    return value if IO === value || StringIO === value
+
+    value = value.load if Step === value
+    if String === value && ! [:path, :file].include?(type) 
+      if Open.exists?(value)
+        Persist.load(value, type) 
+      else
+        Persist.deserialize(value, type)
+      end
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        if Array === value
+          value.collect{|v| format_input(v, m[1].to_sym, options) }
+        end
+      else
+        value
+      end
+    end
+  end
+
+  def assign_inputs(provided_inputs = {})
+    if self.inputs.nil?
+      case provided_inputs
+      when Array
+        return [provided_inputs, provided_inputs]
+      else
+        return [[], []]
+      end
+    end
+
+    input_array = []
+    non_default_inputs = []
+    self.inputs.each_with_index do |p,i|
+      name, type, desc, value, options = p
+      provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
+      provided = format_input(provided, type, options || {})
+      if ! provided.nil? && provided != value
+        non_default_inputs << name
+        input_array << provided
+      else
+        input_array << value
+      end
+    end
+
+    [input_array, non_default_inputs]
+  end
+
+  def digest_inputs(provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    if Array === provided_inputs 
+      Misc.digest(input_array)
+    else
+      Misc.digest(input_array)
+    end
+  end
+  
+  def process_inputs(provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    digest = Misc.digest(input_array)
+    [input_array, non_default_inputs, digest]
+  end
+
+  def save_file_input(orig_file, directory)
+    basename = File.basename(orig_file)
+    digest = Misc.digest(orig_file)
+    if basename.include? '.'
+      basename.sub!(/(.*)\.(.*)/, "\1-#{digest}.\2")
+    else
+      basename += "-#{digest}"
+    end
+    new_file = File.join(directory, 'saved_input_files', basename)
+    relative_file = File.join('.', 'saved_input_files', basename) 
+    Open.link orig_file, new_file
+    relative_file
+  end
+
+  def save_inputs(directory, provided_inputs = {})
+    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    self.inputs.each_with_index do |p,i|
+      name, type, desc, value, options = p
+      next unless non_default_inputs.include?(name)
+      input_file = File.join(directory, name.to_s)
+
+      if type == :file
+        relative_file = save_file_input(input_array[i], directory)
+        Persist.save(relative_file, input_file, type)
+      elsif type == :file_array
+        new_files = input_array[i].collect do |orig_file|
+          save_file_input(orig_file, directory)
+        end
+        Persist.save(new_files, input_file, type)
+      else
+        Persist.save(input_array[i], input_file, type)
+      end
+    end
+  end
+
+  def load_inputs(directory)
+    self.inputs.collect do |p|
+      name, type, desc, value, options = p
+      filename = File.join(directory, name.to_s) 
+      if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
+        Persist.load(filename, type)
+      else
+        value
+      end
+    end
+  end
+end

data/lib/scout/workflow/task.rb

@@ -0,0 +1,141 @@
+require_relative '../meta_extension'
+require_relative 'step'
+require_relative 'task/inputs'
+
+module Task
+  extend MetaExtension
+  extension_attr :name, :type, :inputs, :deps, :directory, :description
+
+  DEFAULT_NAME = "Default"
+
+  class << self
+    attr_accessor :default_directory
+
+    def default_directory
+      @default_directory ||= Path.setup('var/jobs/Task')
+    end
+  end
+
+  def directory
+    @directory ||= Task.default_directory
+  end
+
+  def exec_on(binding = self, *inputs)
+    binding.instance_exec(*inputs, &self)
+  end
+
+  def dependencies(id, provided_inputs, non_default_inputs = [])
+    return [] if deps.nil?
+    dependencies = []
+    
+    provided_inputs ||= {}
+
+    load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
+      task = hash_options[:task] if hash_options.include?(:task)
+      workflow = hash_options[:workflow] if hash_options.include?(:workflow)
+      id = hash_options[:id] if hash_options.include? :id
+
+      hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
+      inputs = IndiferentHash.add_defaults hash_inputs, inputs
+
+      resolved_inputs = {}
+      inputs.each do |k,v|
+        if Symbol === v
+          input_dep = dependencies.select{|d| d.task_name == v}.first
+          resolved_inputs[k] = input_dep || inputs[v] || k
+        else
+          resolved_inputs[k] = v
+        end
+      end
+      workflow.job(task, id, resolved_inputs)
+    end
+
+    deps.each do |workflow,task,options,block=nil|
+      if provided_inputs.include?(overriden = [workflow.name, task] * "#")
+        dep = provided_inputs[overriden]
+        dep = Step.new dep unless Step === dep
+        dep.type = workflow.tasks[task].type
+        dependencies << dep
+        non_default_inputs << overriden
+        next
+      end
+
+      options ||= {}
+      if block
+        inputs = IndiferentHash.add_defaults options.dup, provided_inputs
+
+        res = block.call id, inputs, dependencies
+
+        case res
+        when Step
+          dep = res
+          dependencies << dep
+          dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+          non_default_inputs.concat(dep_non_default_inputs - options.keys)
+        when Hash
+          new_options = res
+          dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
+          dependencies << dep
+          dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+          dep_non_default_inputs -= options.keys 
+          if new_options.include?(:inputs)
+            dep_non_default_inputs -= new_options[:inputs].keys 
+          else
+            dep_non_default_inputs -= new_options.keys
+          end
+          non_default_inputs.concat(dep_non_default_inputs)
+        when Array
+          res.each do |_res|
+            if Hash === _res
+              new_options = _res
+              dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
+              dependencies << dep
+              dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+              dep_non_default_inputs -= options.keys 
+              if new_options.include?(:inputs)
+                dep_non_default_inputs -= new_options[:inputs].keys 
+              else
+                dep_non_default_inputs -= new_options.keys
+              end
+              non_default_inputs.concat(dep_non_default_inputs)
+            else
+              dep = _res
+              dependencies << dep
+              dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+              non_default_inputs.concat(dep_non_default_inputs - options.keys)
+            end
+          end
+        end
+      else
+        inputs = IndiferentHash.add_defaults options.dup, provided_inputs
+        dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
+        dependencies << dep
+        dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
+        non_default_inputs.concat(dep_non_default_inputs - options.keys)
+      end
+    end
+
+    dependencies
+  end
+
+  def job(id = DEFAULT_NAME, provided_inputs = nil )
+    provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
+    provided_inputs = {} if provided_inputs.nil?
+
+    inputs, non_default_inputs, input_hash = process_inputs provided_inputs
+
+    dependencies = dependencies(id, provided_inputs, non_default_inputs)
+
+    non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
+
+    if non_default_inputs.any?
+      hash = Misc.digest(:inputs => input_hash, :non_default_inputs => non_default_inputs, :dependencies => dependencies)
+      Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :dependencies => dependencies)}"
+      id = [id, hash] * "_"
+    end
+
+    path = directory[id]
+
+    Step.new path, inputs, dependencies, &self
+  end
+end