tap 0.7.9

data/lib/tap/support/templater.rb

@@ -0,0 +1,155 @@
+require 'tap/support/template'
+
+module Tap
+  module Support
+    
+    # Templater generates permutations of a hash, treating it as a template for other 
+    # hashes.  Templater looks for various flags on the end of keys to signify operations 
+    # like method execution ('!') and replacement ('=').
+    #
+    #   module TemplateMethods
+    #     def method_call(input)
+    #       self["result"] = "#{input} was processed"
+    #     end
+    #   end
+    #
+    #   t = Templater.new(
+    #        "key"=> "value", 
+    #        "method_call!" => "method input",
+    #        "replacement=" => "replacement input")
+    #   t.run_methods(TemplateMethods)
+    #   t.run_replace do |input|
+    #     "#{input} was replaced"
+    #   end
+    #
+    #   t.templates           
+    #   # => [{"key" => "value", 
+    #         "result" => "method input was processed",
+    #         "replacement" => "replacement input was replaced"}]
+    #
+    # == Creating Template Methods
+    #
+    # The default Template methods may not be sufficient for your needs, but making new
+    # methods (as above) is easy.  Some simple rules apply:
+    #
+    # - Any method can be called from run_methods provided it has a single input.  
+    # - Modules with the methods will extend the hash templates so 'self' indicates the
+    #   template itself.  
+    # - New templates should be added by pushing the template to the templater, which  
+    #   will already be set by the time the method executes 
+    #   (ex: self.templater << new_template).  
+    # 
+    # Also, it's important to note that ANY empty templates at the end of run_methods
+    # are cleared from the templates array.  Thus, if you want to remove a template from
+    # within a method, simply clear self.
+    #
+    # See the Template code for examples.
+    class Templater
+
+      # Convenience method for making a new Templater, running methods
+      # with the input modules, and making replacements using the input
+      # block.  Returns the resulting templates.
+      def self.make_templates(hash, *modules, &block)
+        t = Templater.new(hash)
+        t.run_methods(*modules)
+        t.run_replace(&block)
+        t.templates
+      end
+      
+      attr_reader :templates
+      
+      # Creates a new Templater with the input hash as the initial template.
+      def initialize(hash)
+        @templates = []
+        self << hash
+      end
+      
+      # Shortcut to add a template to the templater, unless the template is empty.
+      def <<(template)
+        self.templates << template unless template.empty? 
+      end
+      
+      # Iterates through the pairs in each template looking for keys matching the
+      # regexp (using =~).  When a matching key is found, it is removed from the 
+      # template and then the template, the key, and the corresponding value are 
+      # passed to the block for further processing. Iteration restarts after each 
+      # match and empty templates are removed upon completion.  All templates are 
+      # extended with Template before iteration.
+      #
+      # Notes:
+      # - run_templater is the foundation of the other run_* methods and may be used
+      #   for custom template processing.
+      # - since =~ is used for comparison, only String keys will be selected
+      #   under normal circumstances.
+      def run_templater(regexp) # :yields: template, key, value
+        templates.each do |template|
+          unless template.respond_to?(:templater)
+            template.extend Template
+            template.templater = self
+          end
+          
+          template.each_pair do |key, value|
+            next unless key =~ regexp
+          
+            # remove the key
+            template.delete(key)
+            yield(template, key, value)
+          
+            # restart the loop as key/value pairs may have changed
+            retry
+          end
+        end
+        
+        templates.delete_if {|t| t.empty?}
+        
+        self
+      end
+      
+      # Iterates through the pairs in each template looking for and executing method calls
+      # as per run_templater. Method calls are indicated by an exclamation attached to the 
+      # key, like 'do_something!'. 
+      #
+      # Templates are extended with the input modules before the method call.  The modules
+      # should provide the methods (an error will be raised if the specified method is not
+      # defined).  Note, templates will by default be extended with the Template module.
+      #
+      #   module MethodModule
+      #     def method_call(input)
+      #       self["result"] = "#{input} was processed"
+      #     end
+      #   end
+      #
+      #   t = Templater.new('method_call!' => 'input')
+      #   t.run_methods(MethodModule)   # calls 'do_something' with the input 'some value',
+      #                                 # after removing the key-value pair
+      #   t.templates                   # => [{"result" => "input was processed"}]
+      #
+      # Notes:
+      # - The method called is the method minus the exclamation, so 'do_something!' calls 
+      #   'do_something' within the template and 'do_something!!' calls 'do_something!'
+      # - Iteration restarts after each method call; inserted method calls will be executed
+      def run_methods(*modules)
+        run_templater(/!$/) do |template, key, value|
+          modules.each {|mod| template.extend mod}
+          template.send(key.chop, value) 
+        end
+      end
+  
+      # Iterates through the pairs in each template looking for replacement keys and
+      # executing the block to make the appropriate replacement value, as per 
+      # run_templater.  Replacement keys are indicated by an equals sign attached to 
+      # the key, like 'key='. The equals sign will be chopped off for these keys, and
+      # the value replaced with the return value of the block.
+      #
+      #   t = Templater.new('replacement=' => 'some value')
+      #   t.run_replace {|value| "#{value} was replaced"}                    
+      #   t.templates                 # => [{"replacement" => "some value was replaced"}]
+      #
+      def run_replace # :yields: value
+        run_templater(/=$/) do |template, key, value|
+          template[key.chop] = yield(value)
+        end  
+      end
+    end
+  end
+end

data/lib/tap/support/versions.rb

@@ -0,0 +1,63 @@
+module Tap
+  module Support
+    
+    # Version provides methods for adding, removing, and incrementing versions
+    # at the end of filepaths.  Versions are all formatted like:
+    # 'filepath-version.extension'.
+    #
+    module Versions
+      
+      # Adds a version to the filepath.  Versioned filepaths follow the format: 
+      # 'path-version.extension'.  If no version is specified, then the filepath 
+      # is returned.
+      #
+      #   version("path/to/file.txt", 1.0)  # => "path/to/file-1.0.txt"
+      #
+      def version(path, version)
+        version = version.to_s.strip
+        if version.empty? 
+          path
+        else
+          extname = File.extname(path)
+          path.chomp(extname) + '-' + version + extname
+        end
+      end
+      
+      # Increments the version of the filepath by the specified increment.  
+      #
+      #   increment("path/to/file-1.0.txt", "0.0.1")  # => "path/to/file-1.0.1.txt"
+      #   increment("path/to/file.txt", 1.0)  # => "path/to/file-1.0.txt"
+      #
+      def increment(path, increment)
+        path, version = deversion(path)
+        
+        # split the version and increment into integer arrays of equal length
+        increment, version = [increment, version].collect do |vstr| 
+          begin
+            vstr.to_s.split(/\./).collect {|v| v.to_i}
+          rescue
+            raise "Bad version or increment: #{vstr}"
+          end
+        end
+        version.concat Array.new(increment.length - version.length, 0) if increment.length > version.length
+
+        # add the increment to version
+        0.upto(version.length-1) do |i|
+          version[i] += (increment[i] || 0)
+        end
+
+        self.version(path, version.join("."))
+      end
+      
+      # Splits the version from the input path, then returns the path and version.  
+      # If no version is specified, then the returned version will be nil.
+      #
+      #   deversion("path/to/file-1.0.txt")  # => ["path/to/file.txt", "1.0"]
+      #   deversion("path/to/file.txt")  # => ["path/to/file.txt", nil]
+      #
+      def deversion(path)
+        path =~ /^(.*)-(\d(\.?\d)*)(.*)?/ ? [$1 + $4, $2] : [path, nil]
+      end
+    end
+  end
+end

data/lib/tap/task.rb

@@ -0,0 +1,448 @@
+require 'monitor'
+
+module Tap
+
+  # == Overview
+  #
+  # Tasks are the basic executable unit of Tap.  When an App executes
+  # a Task, the Task will be passed an array of inputs which it then
+  # processes into outputs.  Tasks are joined into workflows using 
+  # conditions that define when a set of inputs are ready for execution,
+  # and what to do with the outputs when a the Task completes. Tasks 
+  # therefore fundamentally consist of a condition_block, a process 
+  # method, and an on_complete_block.
+  # 
+  # Tasks are configurable.  By default each task will be configured
+  # with the class default_config, which can be set when the class is
+  # defined. 
+  #
+  #   class ConfiguredTask < Tap::Task
+  #     set_default_config :one => 'one', :two => 'two'
+  #   end
+  # 
+  #   t = ConfiguredTask.new
+  #   t.config            # => {:one => 'one', :two => 'two'}
+  #  
+  # Tasks also have a name (based on the class by default) which is
+  # used during initialization to lookup and merge any configurations 
+  # in the corresponding config_file.  Additional, overriding 
+  # configurations may be provided during initialization as well.
+  #
+  #   t.name              # => "configured_task"
+  #   t.app[:config]      # => "/path/to/app/config"
+  #   t.config_file       # => "/path/to/app/config/configured_task.yml"
+  # 
+  #   # [/path/to/app/config/example.yml]
+  #   # one: ONE
+  #  
+  #   t = ConfiguredTask.new "example", :three => 'three'
+  #   t.name              # => "example"
+  #   t.config_file       # => "/path/to/app/config/example.yml"
+  #   t.config            # => {:one => 'ONE', :two => 'two', :three => 'three'}
+  #
+  # === Batches
+  #
+  # Tasks are designed to facilitate batch processing of inputs.  When
+  # a task queues inputs to itself, all inputs are collected into a 
+  # single batch until execution at which time all inputs are passed
+  # to the task at once.  If the task is iterative (the default), then
+  # each of these inputs is processed individually:
+  #
+  #   runlist = []
+  #   t1 = Task.new {|task, input| runlist << input}
+  #   t1.queue 1
+  #   t1.queue 2,3
+  #   t1.app.run
+  #
+  #   runlist            # => [1,2,3]
+  #
+  # However, tasks are also designed to facilitate batch processing 
+  # using a variety (ie batch) of configurations.  When a task is
+  # queued, each task in the batch attribute will be queued as well.
+  #
+  #   runlist = []
+  #   t1.batch                      # => [t1]
+  #   t2 = t1.create_batch_task            
+  #   t1.batch                      # => [t1, t2]
+  #   
+  #   t1.queue 1
+  #   t2.queue 2,3
+  #   t1.app.run
+  #   
+  #   runlist                       # => [1,2,3, 1,2,3]        
+  #   
+  # In the example, runlist reflects that t1 and t2 were run in succession 
+  # with the same [1,2,3] inputs.  Configuration files may specify an array
+  # of configuration templates; each is translated into a batched task.
+  #
+  #   # [/path/to/app/config/batch.yml]
+  #   # - one: ONE
+  #   # - one: ANOTHER ONE
+  #
+  #   t = ConfiguredTask.new "batch"
+  #   t.batch.size        # => 2
+  #   t1, t2 = t.batch
+  #
+  #   t1.name             # => "batch"
+  #   t1.config_template  # => {:one => 'ONE'}
+  #   t1.config           # => {:one => 'ONE', :two => 'two'}
+  #
+  #   t2.name             # => "batch"
+  #   t2.config_template  # => {:one => 'ANOTHER ONE'}
+  #   t2.config           # => {:one => 'ANOTHER ONE', :two => 'two'}
+  #
+  # In addition, configuration files are processed to make the definition
+  # of configuration templates more concise and powerful 
+  # (see Tap::Support::Templater for more details).  The most basic form
+  # merges each variation with the rest of the file to produce the final
+  # array of templates.
+  # 
+  #   # [/path/to/app/config/template.yml]
+  #   # one: ONE
+  #   # variations!:
+  #   #   - two: TWO
+  #   #   - three: THREE
+  #
+  #   t = ConfiguredTask.new "template"
+  #   t.batch.size        # => 2
+  #   t1, t2 = t.batch
+  #
+  #   t1.config           # => {:one => 'ONE', :two => 'TWO'}
+  #   t2.config           # => {:one => 'ONE', :two => 'two', :three => 'THREE'}
+  #
+  # All together, these features make it easy to process a batch of inputs
+  # using a batch of configurations -- all encapsulated in a workflow-ready
+  # architecture.
+  class Task < Monitor
+    write_inheritable_hash(:default_config, {})  
+    class_inheritable_hash(:default_config)  
+
+    write_inheritable_attribute(:iterative, true)  
+    class_inheritable_reader(:iterative)
+ 
+    class << self
+  
+      # Sets the defualt configurations for the task class.  Configurations
+      # are inherited, but can be overridden or added to in subclasses.  Use
+      # options (:make_accessors, :make_readers, :make_writers) to create 
+      # accessors for configurations.
+      #
+      #   class SomeTask < Tap::Task
+      #     set_default_config({
+      #       :key => 'value'},
+      #       :make_accessors => true)
+      #   end
+      #
+      #   t = SomeTask.new
+      #   t.key               # => 'value'
+      #   t.key = 'another'
+      #   t.config            # => {:key => 'another'}
+      #
+      def set_default_config(hash={}, options={})
+        keys = hash.keys
+        self.write_inheritable_hash(:default_config, hash) 
+       
+        self.config_accessor(*keys) if options[:make_accessors]
+        self.config_reader(*keys) if options[:make_readers]
+        self.config_writer(*keys) if options[:make_writers]
+      end
+      
+      # Creates a configuration writer for the input keys.  Works like
+      # attr_writer, except the value is written to config, rather than
+      # a local variable.
+      def config_writer(*keys)
+        keys.each do |key|
+          define_method("#{key}=") do |value|
+            config[key] = value
+          end
+        end
+      end
+      
+      # Creates a configuration reader for the input keys.  Works like
+      # attr_reader, except the value is read from config, rather than
+      # a local variable.
+      def config_reader(*keys)
+        keys.each do |key|
+          define_method(key) do
+            config[key]
+          end
+        end
+      end
+      
+      # Creates configuration accessors for the input keys.  Works like
+      # attr_accessor, except the value is read from and written to config, 
+      # rather than a local variable.
+      def config_accessor(*keys)
+        config_writer(*keys)
+        config_reader(*keys)
+      end
+      
+      # Sets the default iteration behavior for a task class 
+      # to iterate inputs during task execution.
+      def iterate
+        self.write_inheritable_attribute(:iterative, true)  
+      end
+    
+      # Sets the default iteration behavior for a task class 
+      # to not iterate inputs during task execution.
+      def do_not_iterate
+        self.write_inheritable_attribute(:iterative, false)  
+      end
+    end
+   
+    attr_reader :name, :config, :config_template, :app, :batch, :condition_block, :task_block, :on_complete_block, :results
+    attr_accessor :multithread, :iterate
+    
+    # Creates a new Task with the specified attributes.
+    def initialize(name=nil, config=nil, app=App.instance, &task_block)
+      super() # required for Monitor
+      
+      self.name = name.nil? ? self.class.to_s.underscore : name
+      self.app = app
+      
+      self.condition_block = nil
+      self.task_block = task_block
+      self.on_complete_block = nil
+      self.results = [] 
+      self.multithread = false 
+      self.iterate = self.class.iterative
+      self.batch = []
+ 
+      # collect all configuration templates from the app
+      templates = []
+      app.config_templates(self.config_file).each do |template|
+        templates.concat make_config_templates(template)
+      end
+      
+      # be sure there is at least one template and
+      # add a task to batch for every template
+      templates << {} if templates.empty?
+      templates.each do |template|
+        self.create_batch_task(template, config)
+      end
+    end
+    
+    # Creates a batched task based on the config_template and overrides,
+    # and adds the task to batch.  The batched task will be a duplicate 
+    # of the current task but with new configurations.  
+    #
+    # This method can be overridden to initialize variables that are 
+    # specific to a batch task, for instance variables depending on
+    # batch_index.
+    def create_batch_task(config_template={}, overrides={})
+      task = (self.batch.empty? ? self : self.dup)
+      task.config_template = config_template.symbolize_keys
+      task.batch << task
+      task.config = overrides
+      task
+    end
+    
+    # Returns true if the batch size is greater than one 
+    # (the one being the current task itself).  
+    def batched?
+      batch.length > 1
+    end
+    
+    # Returns the index of the current task in batch.
+    def batch_index
+      batch.index(self)
+    end
+    
+    # Returns true if multithread is true.
+    def multithread?
+      multithread
+    end
+    
+    # Returns true if iterate is true.  If iterate has not been set for the task, 
+    # then iterate? returns the value of the class iterative variable.
+    def iterate?
+      iterate
+    end
+        
+    # Returns the path to the configuration file used to make the task configuration
+    # templates.  By default this is the YAML file for the task name relative to the
+    # application config directory:
+    #
+    #   t.app['config']         # => '/path/to/config'
+    #   t.name                  # => 'some/task'
+    #   t.config_file           # => '/path/to/config/some/task.yml'
+    #
+    def config_file
+      # since this is called during initialization before they
+      # are set, take care that this does not depend on batch,
+      # config, or config_template
+      app.filepath('config', self.name + ".yml")
+    end
+
+    # Configures the task with the given configuration overrides.  A Task has three
+    # configuration sources: the class default_config, the config_template loaded from
+    # config_file, and the inputs to this method. Configurations from these 
+    # sources are merged as:  default_config.merge(config_template).merge(overrides)
+    #
+    # Configurations are symbolized before they are merged.  If overrides is nil, then
+    # they will be treated as an empty hash.
+    def config=(overrides)
+      overrides = overrides.nil? ? {} : overrides.symbolize_keys 
+      @config = self.class.default_config.merge(config_template).merge(overrides)
+     
+      self.config
+    end
+    
+    # Sets a condition block for the task.  Raises an error if condition_block is 
+    # already set, unless override = true.
+    def condition(override=false, &block) # :yields: self, inputs
+      raise "Condition for task already set: #{name}" unless condition_block.nil? || override
+      self.condition_block = block
+    end
+    
+    # Sets a block to execute when the task completes execution.  Raises an error 
+    # if on_complete_block is already set, unless override = true.
+    def on_complete(override=false, &block) # :yields: results
+      raise "On complete for task already set: #{name}" unless on_complete_block.nil? || override
+      self.on_complete_block = block
+    end
+    
+    # Returns true if no condition block is specified, or the condition
+    # block evaluates to true with the given inputs.
+    def executable?(inputs)
+      condition_block ? condition_block.call(self, inputs) : true
+    end
+    
+    # Execute the actions associated with this task.  Returns an array of Audits; 
+    # one for each input value.  Raises an error if the task is not executable with 
+    # the inputs, and executes the on_complete block when finished.
+    #
+    # Execute is synchronized such that a task can only execute on one thread at a 
+    # time.
+    def execute(*inputs)
+      synchronize do
+        self.results = nil
+        raise "Condition not met." unless executable?(inputs)
+
+        inputs = case
+        when inputs.empty? then inputs
+        when iterate? then Support::Audit.register(*inputs)
+        else [Support::Audit.merge(*inputs)]
+        end
+        
+        before_execute
+        begin 
+          self.results = inputs.each do |input|
+            output = process( input._current )
+            input._record(self, output)
+          end
+        rescue
+          on_execute_error($!)
+        end
+        after_execute
+        
+        on_complete_block.call(results) if on_complete_block
+        results
+      end
+    end
+    
+    # The method for processing inputs into outputs.  Override this method in
+    # subclasses to provide class-specific process logic.  By default the 
+    # input is passed to the task_block (provided during initialization).
+    # Simply returns the input if no task_block is set.
+    def process(input)
+      task_block.nil? ? input : task_block.call(self, input)
+    end
+    
+    # Enqueues self to the app queue with the inputs.  
+    def enq(*inputs)
+      app.queue.enq(self, *inputs)
+    end
+
+    # Logs the inputs to the application logger (via app.log)
+    def log(action, msg="", level=Logger::INFO)
+      # TODO - add a task identifier?
+      app.log(action, msg, level)
+    end
+    
+    protected
+
+    attr_writer :name, :config_template, :app, :batch, :condition_block, :task_block, :on_complete_block, :results
+
+    # Hook to execute code before inputs are processed.
+    def before_execute() end
+    
+    # Hook to execute code after inputs are processed, but before the on_complete block.
+    def after_execute() end
+
+    # Hook to handle unhandled errors from processing inputs on a task level.  
+    # By default on_execute_error simply re-raises the unhandled error.
+    def on_execute_error(err)
+      raise err
+    end
+
+    # Hook to provide class-specific processing of the config_file template
+    # (ie app.config_template(config_file)).  By default the template is 
+    # processed using Support::Templater.make_templates.
+    def make_config_templates(template)
+      Support::Templater.make_templates(template) 
+      # this would be a security hole... can't do it.
+      # if you can set :data then you could read 
+      # any accessible file.
+      #{ |filename| app.read(:data, filename) }
+    end
+
+  end
+end
+
+class Hold # :nodoc:
+  attr_accessor :help_doc
+  
+  def register_doc(*section_headers)
+    section_headers << "Command Line Usage" if section_headers.empty?
+    self.help_doc = [caller.first.sub(/:\d+$/, ''), section_headers.collect {|section| section.strip}]
+  end
+  
+  def help
+    return "No help available for #{self}" unless help_doc
+    
+    lines = []
+    help_file, sections = help_doc
+    File.open(help_file) do |file|
+      in_section = nil
+      while line = file.gets
+        next unless line =~ /^#(.*)/
+        line = $1.strip
+        
+        if line =~ /^=+(.*)/
+          line = $1.strip
+          in_section = sections.delete(line) 
+          line += ":"
+        else
+          line = "  " + line
+        end
+        
+        lines << line if in_section
+      end
+    end
+  
+    return lines.join("\n")
+  end
+    
+    # Queues self to app with the input.
+    #def <<(input)
+      #queue(input)
+    #end
+    
+   # Ticks the monitor by printing '.', or the specified message.
+   # Only ticks if called from within a monitor block.
+   #def tick_monitor(action=nil, msg="", level=Logger::INFO)
+    # app.tick_monitor(action, msg, level)
+   #end
+   
+   # Prints the hash of statistics if called within a monitor block.
+   #def monitor_stats(hash)
+    # app.monitor_stats(hash)
+   #end
+   
+   # Times the execution of the block and enables the various monitor
+   # methods (tick_monitor, monitor_stats).  
+  # def monitor(action="beginning", msg="", &block)
+   #  app.monitor(action,msg, &block)
+   #end
+end