rbbt-util 5.1.0 → 5.2.0

data/lib/rbbt/util/named_array.rb

@@ -8,35 +8,42 @@ module NamedArray
   attr_accessor :fields
   attr_accessor :key
   attr_accessor :entity_options
+  attr_accessor :entity_templates
 
-  def self.setup(array, fields, key = nil, entity_options = nil)
+  def entity_templates
+    @entity_templates ||= {}
+  end
+
+  def self.setup(array, fields, key = nil, entity_options = nil, entity_templates = nil)
     array.extend NamedArray unless NamedArray === array
     array.fields = fields
     array.key = key
-    array.entity_options = entity_options
+    array.entity_options = entity_options unless entity_options.nil?
+    array.entity_templates = entity_templates unless entity_templates.nil?
     array
   end
 
   def prepare_entity(entity, field, options = {})
     return entity if entity.nil?
     return entity unless defined? Entity
-    @entity_templates ||= {}
-    if (template = @entity_templates[field])
+    template = entity_templates[field]
+    entity_templates ||= {}
+    if template
       entity = template.annotate(entity.frozen? ? entity.dup : entity)
       entity.extend AnnotatedArray if Array === entity
       entity
     else
-      if @entity_templates.include? field
+      if entity_templates.include? field
         entity
       else
         template = Misc.prepare_entity("TEMPLATE", field, options)
         if Annotated === template
-          @entity_templates[field] = template
+          entity_templates[field] = template
           entity = template.annotate(entity.frozen? ? entity.dup : entity)
           entity.extend AnnotatedArray if Array === entity
           entity
         else
-          @entity_templates[field] = nil
+          entity_templates[field] = nil
           entity
         end
       end

data/lib/rbbt/util/open.rb

@@ -180,6 +180,7 @@ module Open
   end
 
   def self.open(url, options = {})
+    return url if IO === url
     options = Misc.add_defaults options, :noz => false, :mode => 'r'
 
     mode = Misc.process_options options, :mode
@@ -283,7 +284,7 @@ module Open
           f.flock(File::LOCK_UN)
         end
       rescue Exception
-        FileUtils.rm file if File.exists? file
+        FileUtils.rm_rf file if File.exists? file
         raise $!
       end
       content.close

data/lib/rbbt/util/tmpfile.rb

@@ -28,8 +28,11 @@ module TmpFile
     File.join(TMPDIR, random_name(s,max))
   end
 
-  def self.with_file(content = nil, erase = true)
+  def self.with_file(content = nil, erase = true, options = {})
     tmpfile = tmp_file
+    if options[:extension]
+      tmpfile += ".#{options[:extension]}"
+    end
 
     File.open(tmpfile, 'w') do |f| f.write content end if content != nil
 
@@ -39,4 +42,16 @@ module TmpFile
 
     result
   end
+
+  def self.with_dir(erase = true, options = {})
+    tmpdir = tmp_file
+
+    FileUtils.mkdir_p tmpdir
+
+    result = yield(tmpdir)
+
+    FileUtils.rm_rf tmpdir if File.exists?(tmpdir) and erase
+
+    result
+  end
 end

data/lib/rbbt/workflow.rb

@@ -1,14 +1,33 @@
+require 'rbbt/workflow/definition'
 require 'rbbt/workflow/task'
 require 'rbbt/workflow/step'
-require 'rbbt/workflow/annotate'
 require 'rbbt/workflow/accessor'
 
 module Workflow
+  def self.resolve_locals(inputs)
+    inputs.each do |name, value|
+      if value =~ /^local:(.*?):(.*)/ or 
+        (Array === value and value.length == 1 and value.first =~ /^local:(.*?):(.*)/) or
+        (TSV === value and value.size == 1 and value.keys.first =~ /^local:(.*?):(.*)/)
+        task_name = $1
+        jobname = $2
+        value = load_id(File.join(task_name, jobname)).load
+      end
+      inputs[name] = value
+    end 
+  end
+
+  #{{{ WORKFLOW MANAGEMENT 
   class << self
     attr_accessor :workflows
   end
   self.workflows = []
 
+  def self.extended(base)
+    self.workflows << base
+    base.libdir = Path.caller_lib_dir.tap{|p| p.resource = base}
+  end
+
   def self.require_remote_workflow(wf_name, url)
     require 'rbbt/workflow/rest/client'
     eval "Object::#{wf_name} = RbbtRestClient.new '#{ url }', '#{wf_name}'"
@@ -95,10 +114,10 @@ module Workflow
     rescue Exception
       Log.debug $!.message 
       Log.debug $!.backtrace.first
-      raise "Workflow not found: #{ wf_name }" if wf_name == Misc.humanize(wf_name)
-      Log.debug "Trying with humanized: '#{Misc.humanize wf_name}'"
+      raise "Workflow not found: #{ wf_name }" if wf_name == Misc.snake_case(wf_name)
+      Log.debug "Trying with humanized: '#{Misc.snake_case wf_name}'"
       begin
-        require_local_workflow(Misc.humanize(wf_name))
+        require_local_workflow(Misc.snake_case(wf_name))
       rescue Exception
         Log.debug $!.message
         raise "Workflow not found: #{ wf_name }"
@@ -106,121 +125,65 @@ module Workflow
     end
   end
 
-  def self.extended(base)
-    if not base.respond_to? :workdir
-      base.extend AnnotatedModule
-      class << base
-        attr_accessor :libdir, :workdir, :tasks, :task_dependencies, :task_description, :dependencies, :asynchronous_exports, :synchronous_exports, :exec_exports, :last_task
- 
-        alias prev_workflow_extended extended if methods.include? "extended"
-
-        def extended(object)
-          self.send(:prev_workflow_extended, object) if methods.include? "prev_workflow_extended"
-          object.extend Workflow unless Workflow === object
-
-          object.tasks.merge! self.tasks
-          object.task_dependencies.merge! self.task_dependencies
-        end
-
-        def dependencies
-          i = @dependencies; @dependencies = []; i
-        end
-
-        def task_dependencies
-          IndiferentHash.setup(@task_dependencies || {})
-        end
-
-        def tasks
-          IndiferentHash.setup(@tasks || {})
-        end
-      end
+  attr_accessor :libdir, :workdir 
+  attr_accessor :helpers, :tasks
+  attr_accessor :task_dependencies, :task_description, :last_task 
+  attr_accessor :asynchronous_exports, :synchronous_exports, :exec_exports
 
-      if defined? Rbbt
-        base.workdir = Rbbt.var.jobs.find
-      else
-        base.workdir = Path.setup('var/jobs')
-      end
-      base.tasks = {}
-      base.dependencies = []
-      base.task_dependencies = {}
-      base.task_description = {}
-      base.asynchronous_exports = []
-      base.synchronous_exports = []
-      base.exec_exports = []
-      base.libdir = Path.caller_lib_dir
-    end
-    self.workflows << base
-  end
+  #{{{ ATTR DEFAULTS
 
-  # {{{ Task definition helpers
+  def workdir
+    @workdir ||= if defined? Rbbt
+                   Rbbt.var.jobs[self].find
+                 else
+                   Path.setup('var/jobs')
+                 end
+  end
 
-  def task(name, &block)
-    if Hash === name
-      result_type = name.first.last
-      name = name.first.first
-    else
-      result_type = :marshal
-    end
+  def libdir
+    @libdir = Path.caller_lib_dir if @libdir.nil?
+    @libdir 
+  end
 
-    name = name.to_sym
-
-    block = self.method(name) unless block_given?
-
-    result_type = result_type
-    annotations = {
-      :name => name,
-      :inputs => inputs,
-      :description => description,
-      :input_types => input_types,
-      :result_type => Array == result_type ? result_type.to_sym : result_type,
-      :input_defaults => input_defaults,
-      :input_descriptions => input_descriptions,
-      :input_options => input_options,
-      :result_description => result_description
-    }
-    
-    task = Task.setup(annotations, &block)
+  def helpers
+    @helpers ||= {}
+  end
 
-    @last_task = task
-    @tasks[name] = task
-    @task_dependencies[name] = dependencies
+  def tasks
+    @tasks ||= {} 
   end
 
-  def export_exec(*names)
-    @exec_exports.concat names
+  def task_dependencies
+    @task_dependencies ||= {} 
   end
 
-  def export_asynchronous(*names)
-    @asynchronous_exports.concat names
+  def task_description
+    @task_description ||= {}
   end
 
-  def export_synchronous(*names)
-    @synchronous_exports.concat names
+  def asynchronous_exports
+    @asynchronous_exports ||= []
   end
 
-  # {{{ Job management
+  def synchronous_exports
+    @synchronous_exports ||= []
+  end
 
-  def resolve_locals(inputs)
-    inputs.each do |name, value|
-      if value =~ /^local:(.*?):(.*)/ or 
-        (Array === value and value.length == 1 and value.first =~ /^local:(.*?):(.*)/) or
-        (TSV === value and value.size == 1 and value.keys.first =~ /^local:(.*?):(.*)/)
-        task_name = $1
-        jobname = $2
-        value = load_id(File.join(task_name, jobname)).load
-      end
-      inputs[name] = value
-    end 
+  def exec_exports
+    @exec_exports ||= []
   end
 
+  # {{{ JOB MANAGEMENT
+
   def job(taskname, jobname = nil, inputs = {})
+    taskname = taskname.to_sym
     jobname = "Default" if jobname.nil? or jobname.empty?
     task = tasks[taskname]
     raise "Task not found: #{ taskname }" if task.nil?
 
     IndiferentHash.setup(inputs)
 
-    resolve_locals(inputs)
+    Workflow.resolve_locals(inputs)
 
     dependencies = real_dependencies(task, jobname, inputs, task_dependencies[taskname] || [])
 
@@ -230,9 +193,8 @@ module Workflow
 
     step = Step.new step_path, task, input_values, dependencies
 
-    helpers = @helpers
-    (class << step; self; end).class_eval do
-      helpers.each do |name, block|
+    helpers.each do |name, block|
+      (class << step; self; end).instance_eval do
         define_method name, &block
       end
     end
@@ -242,13 +204,13 @@ module Workflow
 
   def load_step(path)
     task = task_for path
-    Step.new path, tasks[task]
+    Step.new path, tasks[task.to_sym]
   end
 
   def load_id(id)
     path = File.join(workdir, id)
     task = task_for path
-    step = Step.new path, tasks[task]
+    step = Step.new path, tasks[task.to_sym]
     if step.info.include? :dependencies
       step.dependencies = step.info[:dependencies].collect do |task, job|
         load_id(File.join(task.to_s, job))

data/lib/rbbt/workflow/accessor.rb

@@ -11,6 +11,10 @@ class Step
     name.sub(/(.*)_.*/, '\1')
   end
 
+  def task_name
+    @task_name ||= task.name
+  end
+
   # {{{ INFO
 
   def info_file
@@ -71,8 +75,13 @@ class Step
   end
 
   def done?
-    status = info[:status]
-    status == :done or status == :error
+    path.exists?
+  end
+
+  def running?
+    return nil if not File.exists? info_file
+    return nil if info[:pid].nil?
+    return Misc.pid_exists? info[:pid]
   end
 
   def error?
@@ -166,6 +175,7 @@ module Workflow
   end
 
   def task_info(name)
+    name = name.to_sym
     task = tasks[name]
     description = task.description
     result_description = task.result_description
@@ -187,7 +197,7 @@ module Workflow
              end
 
     
-    dependencies = @task_dependencies[name].select{|dep| String === dep or Symbol === dep}
+    dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
     { :id => File.join(self.to_s, name.to_s),
       :description => description,
       :export => export,
@@ -213,23 +223,23 @@ module Workflow
   end
 
   def rec_inputs(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat tasks[tn].inputs}
+    [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat tasks[tn.to_sym].inputs}
   end
 
   def rec_input_defaults(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_defaults}
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn.to_sym].input_defaults}
   end
 
   def rec_input_types(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_types}
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn.to_sym].input_types}
   end
 
   def rec_input_descriptions(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_descriptions}
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn.to_sym].input_descriptions}
   end
 
   def rec_input_options(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_options}
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn.to_sym].input_options}
   end
 
 
@@ -240,7 +250,7 @@ module Workflow
       when Step === dependency
         dependency
       when Symbol === dependency
-        job dependency, jobname, inputs
+        job(dependency, jobname, inputs)
       when Proc === dependency
         dependency.call jobname, inputs
       end

data/lib/rbbt/workflow/annotate.rb

@@ -1,79 +1,48 @@
 module AnnotatedModule
-  def self.extended(base)
-    if not base.respond_to? :inputs
-      class << base
-        attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :input_options, :result_description, :helpers
 
-        def description
-          i = @description; @description = ""; i
+  def self.add_consummable_annotation(target, *annotations)
+    if annotations.length == 1 and Hash === annotations.first
+      annotations.first.each do |annotation, default|
+        target.send(:attr_accessor, annotation)
+        target.send(:define_method, "consume_#{annotation}") do
+          value = instance_variable_get("@#{annotation}") || default.dup
+          instance_variable_set("@#{annotation}", default.dup)
+          value
         end
-
-        def inputs
-          i = @inputs; @inputs = []; i
-        end
-
-        def input_types
-          i = @input_types; @input_types = {}; i
-        end
-
-        def input_descriptions
-          i = @input_descriptions; @input_descriptions = {}; i
-        end
-
-        def input_defaults
-          i = @input_defaults; @input_defaults = {}; i
-        end
-
-        def input_options
-          i = @input_options; @input_options = {}; i
-        end
-
-        def description
-          i = @description; @description = ""; i
-        end
-
-        def result_description
-          i = @result_description; @result_description = nil; i
+      end
+    else
+      annotations.each do |annotation|
+        target.send(:attr_accessor, annotation)
+        target.send(:define_method, "consume_#{annotation}") do
+          value = instance_variable_get("@#{annotation}")
+          instance_variable_set("@#{annotation}", nil)
         end
       end
-
-      base.description = ""
-      base.inputs = []
-      base.input_types = {}
-      base.input_descriptions = {}
-      base.input_defaults = {}
-      base.input_options = {}
-      base.helpers = {}
-
     end
   end
 
-  def helper(name, &block)
-    @helpers[name] = block
-  end
-
-  def returns(text)
-    @result_description = text
-  end
-
-  def desc(description)
-    @description = description
-  end
-
-  def dep(*dependencies, &block)
-    dependencies << block if block_given?
-    @dependencies.concat dependencies
-  end
+  add_consummable_annotation(self,
+    :description        => "",
+    :inputs             => [],
+    :input_types        => {},
+    :input_descriptions => {},
+    :input_defaults     => {},
+    :input_options      => {})
 
   def input(name, type = nil, desc = nil, default = nil, options = nil)
     name = name.to_sym
     type = type.to_sym
-    @inputs << name
-    @input_types[name] = type unless type.nil?
+
+    @inputs             = [] if @inputs.nil?
+    @input_types        = {} if @input_types.nil?
+    @input_descriptions = {} if @input_descriptions.nil?
+    @input_defaults     = {} if @input_defaults.nil?
+    @input_options      = {} if @input_options.nil?
+
+    @inputs                   << name
+    @input_types[name]        = type unless type.nil?
     @input_descriptions[name] = desc unless desc.nil?
-    @input_defaults[name] = default unless default.nil?
-    @input_options[name] = options unless options.nil?
+    @input_defaults[name]     = default unless default.nil?
+    @input_options[name]      = options unless options.nil?
   end
 end
-
-