rbbt-util 3.2.1 → 4.0.0

data/lib/rbbt/util/open.rb

@@ -80,7 +80,7 @@ module Open
   # Cache
 
   def self.in_cache(url, options = {})
-    digest = Digest::MD5.hexdigest([url, options["--post-data"]].inspect)
+    digest = Digest::MD5.hexdigest([url, options["--post-data"], (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
 
     filename = File.join(REMOTE_CACHEDIR, digest)
     if File.exists? filename
@@ -91,7 +91,7 @@ module Open
   end
   
   def self.add_cache(url, data, options = {})
-    digest = Digest::MD5.hexdigest([url, options["--post-data"]].inspect)
+    digest = Digest::MD5.hexdigest([url, options["--post-data"], (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
     Misc.sensiblewrite(File.join(REMOTE_CACHEDIR, digest), data)
   end
 
@@ -181,6 +181,13 @@ module Open
     else
       io
     end
+
+    class << io;
+      attr_accessor :filename
+    end
+
+    io.filename = url.to_s
+    io
   end
 
   def self.can_open?(file)

data/lib/rbbt/util/persistence.rb

@@ -21,7 +21,7 @@ module Persistence
   end
  
   def self.get_persistence_file(file, prefix, options = {})
-    persistence_dir = Misc.process_options options, :persistence_dir
+    persistence_dir = Misc.process_options options, :dir
     persistence_dir ||= CACHEDIR
 
     if options.include? :filters

data/lib/rbbt/util/task/job.rb

@@ -32,7 +32,8 @@ class Task
 
     def previous_jobs_rec
       return [] if previous_jobs.nil?
-      previous_jobs + previous_jobs.collect{|job| job.previous_jobs_rec}.flatten
+      prev = previous_jobs + previous_jobs.collect{|job| job.previous_jobs_rec}.flatten
+      NamedArray.name prev, prev.collect{|job| job.task.name}
     end
 
     def previous_jobs=(previous_jobs)
@@ -221,6 +222,7 @@ class Task
     def run
       return self if recursive_done?
       begin
+        FileUtils.rm info_file if File.exists? info_file
         step(:started)
         start
         step(:done)

data/lib/rbbt/workflow.rb

@@ -0,0 +1,193 @@
+require 'rbbt/workflow/task'
+require 'rbbt/workflow/step'
+require 'rbbt/workflow/annotate'
+require 'rbbt/workflow/accessor'
+
+module Workflow
+  class << self
+    attr_accessor :workflows
+  end
+  self.workflows = []
+
+  def self.require_workflow(wf_name, wf_dir = nil)
+    require 'rbbt/resource/path'
+
+    if File.exists?(wf_name) or File.exists?(wf_name + '.rb')
+      $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
+      require wf_name
+      return
+    end
+
+    wf_dir ||= case
+               when File.exists?(File.join(File.dirname(Path.caller_lib_dir), wf_name))
+                 dir = File.join(File.dirname(Path.caller_lib_dir), wf_name)
+                 Log.debug "Loading workflow from lib dir: #{dir}"
+                 dir
+                 File.join(File.dirname(Path.caller_lib_dir), wf_name)
+               when defined? Rbbt
+                 if Rbbt.etc.workflow_dir.exists?
+                   dir = File.join(Rbbt.etc.workflow_dir.read.strip, wf_name)
+                   Log.debug "Loading workflow from etc dir: #{dir}"
+                   dir
+                 else
+                   dir = Rbbt.workflows[wf_name]
+                   Log.debug "Loading workflow from main dir: #{dir}"
+                   dir
+                 end
+               else
+                 dir = File.join(ENV["HOME"], '.workflows')
+                 Log.debug "Loading workflow from home dir: #{dir}"
+                 dir
+               end
+
+    wf_dir = Path.setup(wf_dir)
+
+    $LOAD_PATH.unshift(File.join(File.dirname(wf_dir["workflow.rb"].find), 'lib'))
+    require wf_dir["workflow.rb"].find
+  end
+
+  def self.extended(base)
+    if not base.respond_to? :workdir
+      base.extend AnnotatedModule
+      class << base
+        attr_accessor :libdir, :workdir, :tasks, :task_dependencies, :task_description, :dependencies, :asynchronous_exports, :synchronous_exports, :exec_exports, :last_task
+
+        def dependencies
+          i = @dependencies; @dependencies = []; i
+        end
+
+        def task_dependencies
+          IndiferentHash.setup(@task_dependencies || {})
+        end
+ 
+        def tasks
+          IndiferentHash.setup(@tasks || {})
+        end
+      end
+
+      if defined? Rbbt
+        base.workdir = Rbbt.var.jobs.find
+      else
+        base.workdir = Path.setup('var/jobs')
+      end
+      base.tasks = {}
+      base.dependencies = []
+      base.task_dependencies = {}
+      base.task_description = {}
+      base.asynchronous_exports = []
+      base.synchronous_exports = []
+      base.exec_exports = []
+      base.libdir = Path.caller_lib_dir
+    end
+    self.workflows << base
+  end
+
+  # {{{ Task definition helpers
+  
+  def task(name, &block)
+    if Hash === name
+      result_type = name.first.last
+      name = name.first.first
+    else
+      result_type = :marshal
+    end
+
+    name = name.to_sym
+
+    block = self.method(name) unless block_given?
+
+    result_type = result_type
+    task = Task.setup({
+      :name => name,
+      :inputs => inputs,
+      :description => description,
+      :input_types => input_types,
+      :result_type => Array == result_type ? result_type.to_sym : result_type,
+      :input_defaults => input_defaults,
+      :input_descriptions => input_descriptions
+    }, &block)
+
+    @last_task = task
+    @tasks[name] = task
+    @task_dependencies[name] = dependencies
+  end
+
+  def export_exec(*names)
+    @exec_exports.concat names
+  end
+
+  def export_asynchronous(*names)
+    @asynchronous_exports.concat names
+  end
+
+  def export_synchronous(*names)
+    @synchronous_exports.concat names
+  end
+
+  # {{{ Job management
+  
+  def resolve_locals(inputs)
+    inputs.each do |name, value|
+      if value =~ /^local:(.*?):(.*)/ or 
+        (Array === value and value.length == 1 and value.first =~ /^local:(.*?):(.*)/) or
+        (TSV === value and value.size == 1 and value.keys.first =~ /^local:(.*?):(.*)/)
+        task_name = $1
+        jobname = $2
+        value = load_id(File.join(task_name, jobname)).load
+      end
+      inputs[name] = value
+    end 
+  end
+
+  def job(taskname, jobname = nil, inputs = {})
+    jobname ||= "Default"
+    task = tasks[taskname]
+    raise "Task not found: #{ taskname }" if task.nil?
+
+
+    IndiferentHash.setup(inputs)
+
+    resolve_locals(inputs)
+
+
+    dependencies = real_dependencies(task, jobname, inputs, task_dependencies[taskname] || [])
+
+    input_values = task.take_input_values(inputs)
+
+    step_path = step_path taskname, jobname, input_values, dependencies
+
+    step = Step.new step_path, task, input_values, dependencies
+
+    step
+  end
+
+  def load(path)
+    task = task_for path
+    Step.new path, tasks[task]
+  end
+
+  def load_id(id)
+    path = File.join(workdir, id)
+    task = task_for path
+    step = Step.new path, tasks[task]
+    if step.info.include? :dependencies
+      step.dependencies = step.info[:dependencies].collect do |task, job|
+        Step.new File.join(workdir, task.to_s, job), tasks[task]
+      end
+    end
+    step
+  end
+
+  def jobs(task, query = nil)
+    task_dir = File.join(workdir.find, task.to_s)
+    if query.nil?
+      path = File.join(task_dir, "**/*.info")
+    else
+      path = File.join(task_dir, query + "*.info")
+    end
+
+    Dir.glob(path).collect{|f|
+      Misc.path_relative_to(task_dir, f).sub(".info",'')
+    }
+  end
+end

data/lib/rbbt/workflow/accessor.rb

@@ -0,0 +1,249 @@
+require 'rbbt/util/open'
+require 'yaml'
+
+class Step
+
+  def name
+    @path.sub(/.*\/#{Regexp.quote task.name.to_s}\/(.*)/, '\1')
+  end
+
+  def clean_name
+    name.sub(/(.*)_.*/, '\1')
+  end
+
+  # {{{ INFO
+
+  def info_file
+    @path + '.info'
+  end
+
+  def info
+    return {} if not File.exists? info_file
+    YAML.load(Open.open(info_file)) || {}
+  end
+
+  def set_info(key, value)
+    Misc.lock(info_file) do
+      i = info
+      i[key] = value
+      Open.write(info_file, i.to_yaml)
+      value
+    end
+  end
+
+  def status
+    info[:status]
+  end
+
+  def status=(status)
+    set_info(:status, status)
+  end
+
+  def messages
+    info[:messages] || set_info(:messages, [])
+  end
+
+  def message(message)
+    set_info(:messages, messages << message)
+  end
+
+  def log(status, message = nil)
+    if message
+      Log.low "#{ status }: #{ message }"
+    else
+      Log.low "#{ status }"
+    end
+    self.status = status
+    message(message) unless message.nil?
+  end
+
+  def done?
+    info[:status] == :done or info[:status] == :error
+  end
+
+  def error?
+    info[:status] == :error
+  end
+
+  # {{{ INFO
+
+  def files_dir
+    @path + '.files'
+  end
+
+  def files
+    Dir.glob(File.join(files_dir, '*')).collect do |path| File.basename(path) end
+  end
+
+  def file(name)
+    Path.setup(File.join(files_dir, name.to_s))
+  end
+
+  def save_file(name, content)
+    content = case
+              when String === content
+                content
+              when Array === content
+                content * "\n"
+              when TSV === content
+                content.to_s
+              when Hash === content
+                content.collect{|*p| p * "\t"} * "\n"
+              else
+                content.to_s
+              end
+    Open.write(file(name), content)
+  end
+
+  def load_file(name, type = nil, options = {})
+    if type.nil? and name =~ /.*\.(\w+)$/
+      extension = name.match(/.*\.(\w+)$/)[1]
+      case extension
+      when "tc"
+        type = :tc
+      when "tsv"
+        type = :tsv
+      when "list", "ary", "array"
+        type = :array
+      when "yaml"
+        type = :yaml
+      when "marshal"
+        type = :marshal
+      else
+        type = :other
+      end
+    else
+      type ||= :other
+    end
+
+    case type.to_sym
+    when :tc
+      Persist.open_tokyocabinet(file(name), false)
+    when :tsv
+      TSV.open Open.open(file(name)), options
+    when :array
+      Open.read(file(name)).split /\n|,\s*/
+    when :yaml
+      YAML.load(Open.open(file(name)))
+    when :marshal
+      Marshal.load(Open.open(file(name)))
+    else
+      Open.read(file(name))
+    end
+  end
+
+
+end
+
+module Workflow
+  def task_info(name)
+    task = tasks[name]
+    description = task.description
+    result_type = task.result_type
+    inputs = rec_inputs(name)
+    input_types = rec_input_types(name)
+    input_descriptions = rec_input_descriptions(name)
+    input_defaults = rec_input_defaults(name)
+    export = case
+             when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
+               :synchronous
+             when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
+               :asynchronous
+             when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
+               :exec
+             else
+               :none
+             end
+
+    
+    dependencies = @task_dependencies[name].select{|dep| String === dep or Symbol === dep}
+    { :id => File.join(self.to_s, name.to_s),
+      :description => description,
+      :export => export,
+      :inputs => inputs,
+      :input_types => input_types,
+      :input_descriptions => input_descriptions,
+      :input_defaults => input_defaults,
+      :result_type => result_type,
+      :dependencies => dependencies
+    }
+  end
+
+  def rec_dependencies(taskname)
+    if task_dependencies.include? taskname
+      deps = task_dependencies[taskname].select{|dep| String === dep or Symbol === dep}
+      deps.concat deps.collect{|dep| rec_dependencies(dep)}.flatten
+      deps.uniq
+    else
+      []
+    end
+  end
+
+  def rec_inputs(taskname)
+    [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat tasks[tn].inputs}
+  end
+
+  def rec_input_defaults(taskname)
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_defaults}
+  end
+
+  def rec_input_types(taskname)
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_types}
+  end
+
+  def rec_input_descriptions(taskname)
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_descriptions}
+  end
+
+  def real_dependencies(task, jobname, inputs, dependencies)
+    real_dependencies = []
+    dependencies.each do |dependency|
+      real_dependencies << case
+      when Step === dependency
+        dependency
+      when Symbol === dependency
+        job dependency, jobname, inputs
+      when Proc === dependency
+        dependency.call jobname, inputs
+      end
+    end
+    real_dependencies.flatten.compact
+  end
+
+  TAG = :hash
+  def step_path(taskname, jobname, inputs, dependencies)
+    raise "Jobname makes an invalid path: #{ jobname }" if jobname =~ /\.\./
+    if inputs.any? or dependencies.any?
+      tagged_jobname = case TAG
+                       when :hash
+                         jobname + '_' + Misc.digest((inputs + dependencies.collect{|dep| dep.name}).inspect)
+                       else
+                         jobname
+                       end
+    else
+      tagged_jobname = jobname
+    end
+
+    workdir[taskname][tagged_jobname].find
+  end
+
+
+  def id_for(path)
+    if workdir.respond_to? :find
+      workdir_find = workdir.find 
+    else
+      workdir_find = workdir
+    end
+    Misc.path_relative_to workdir_find, path
+  end
+
+  def task_for(path)
+    if workdir.respond_to? :find
+      workdir_find = workdir.find 
+    else
+      workdir_find = workdir
+    end
+ 
+    Misc.path_relative_to(workdir_find, File.dirname(path)).sub(/([^\/]+)\/.*/,'\1')
+  end
+end