tap 0.7.9 → 0.8.0

data/lib/tap/task.rb

@@ -1,32 +1,31 @@
-require 'monitor'
-
 module Tap
-
+  
   # == Overview
   #
   # Tasks are the basic executable unit of Tap.  When an App executes
-  # a Task, the Task will be passed an array of inputs which it then
-  # processes into outputs.  Tasks are joined into workflows using 
-  # conditions that define when a set of inputs are ready for execution,
-  # and what to do with the outputs when a the Task completes. Tasks 
-  # therefore fundamentally consist of a condition_block, a process 
-  # method, and an on_complete_block.
+  # a Task, the Task will be passed inputs which it then processes into
+  # results.  Tasks are joined into workflows using condition blocks  
+  # defining when a set of inputs are ready for execution, and where to
+  # pass results when the Task completes. Tasks fundamentally consist 
+  # of a condition_block, a process method, and an on_complete_block.
   # 
   # Tasks are configurable.  By default each task will be configured
-  # with the class default_config, which can be set when the class is
-  # defined. 
+  # with the default class configurations, which can be set when the 
+  # class is defined. 
   #
   #   class ConfiguredTask < Tap::Task
-  #     set_default_config :one => 'one', :two => 'two'
+  #     config :one, 'one'
+  #     config :two, 'two'
   #   end
   # 
   #   t = ConfiguredTask.new
   #   t.config            # => {:one => 'one', :two => 'two'}
   #  
-  # Tasks also have a name (based on the class by default) which is
-  # used during initialization to lookup and merge any configurations 
-  # in the corresponding config_file.  Additional, overriding 
-  # configurations may be provided during initialization as well.
+  # Tasks also have a name (based on the class name by default).  The
+  # task name is used as a relative filepath from application directories
+  # to assoicated files.  During initialization, the task name is used
+  # to lookup configurations from config_file.  Additional and overriding 
+  # configurations may be provided during initialization.
   #
   #   t.name              # => "configured_task"
   #   t.app[:config]      # => "/path/to/app/config"
@@ -42,38 +41,47 @@ module Tap
   #
   # === Batches
   #
-  # Tasks are designed to facilitate batch processing of inputs.  When
-  # a task queues inputs to itself, all inputs are collected into a 
-  # single batch until execution at which time all inputs are passed
-  # to the task at once.  If the task is iterative (the default), then
-  # each of these inputs is processed individually:
+  # Tasks are designed to facilitate batch processing of inputs.  Each
+  # time a task queues inputs to itself, the inputs are collected into a 
+  # batch.  Upon execution, all inputs are passed to the task at once.  
+  # If the task is iterative (the default), then each of these inputs is 
+  # processed individually.  Otherwise, all inputs are processed as a single
+  # group:
   #
   #   runlist = []
   #   t1 = Task.new {|task, input| runlist << input}
-  #   t1.queue 1
-  #   t1.queue 2,3
+  #   t1.enq 1
+  #   t1.enq 2,3
   #   t1.app.run
-  #
   #   runlist            # => [1,2,3]
   #
-  # However, tasks are also designed to facilitate batch processing 
-  # using a variety (ie batch) of configurations.  When a task is
-  # queued, each task in the batch attribute will be queued as well.
+  #   runlist = []
+  #   t1.iterate = false
+  #   t1.enq 1
+  #   t1.enq 2,3
+  #   t1.app.run
+  #   runlist            # => [[1,2,3]]
+  #
+  # Additionally, tasks are designed to facilitate processing using a batch 
+  # of related tasks.  Often these batches consist of the same task class,
+  # but with a variety of configurations, but they can be assembled in other
+  # ways. When a task is queued, each task in the batch will be queued:
   #
   #   runlist = []
+  #   t1 = Task.new {|task, input| runlist << input}
   #   t1.batch                      # => [t1]
   #   t2 = t1.create_batch_task            
   #   t1.batch                      # => [t1, t2]
   #   
-  #   t1.queue 1
-  #   t2.queue 2,3
+  #   t1.enq 1
+  #   t2.enq 2,3
   #   t1.app.run
-  #   
   #   runlist                       # => [1,2,3, 1,2,3]        
   #   
-  # In the example, runlist reflects that t1 and t2 were run in succession 
-  # with the same [1,2,3] inputs.  Configuration files may specify an array
-  # of configuration templates; each is translated into a batched task.
+  # Here runlist reflects that t1 and t2 were run in succession with the same [1,2,3] 
+  # inputs. Configuration batches are automatically generated when task.config_file 
+  # specifies an array of configurations; each configuration is translated into a 
+  # batched task.
   #
   #   # [/path/to/app/config/batch.yml]
   #   # - one: ONE
@@ -91,11 +99,10 @@ module Tap
   #   t2.config_template  # => {:one => 'ANOTHER ONE'}
   #   t2.config           # => {:one => 'ANOTHER ONE', :two => 'two'}
   #
-  # In addition, configuration files are processed to make the definition
-  # of configuration templates more concise and powerful 
-  # (see Tap::Support::Templater for more details).  The most basic form
-  # merges each variation with the rest of the file to produce the final
-  # array of templates.
+  # Task processes configuration files to make the definition of configuration 
+  # templates more DRY and powerful (see Tap::Support::Templater for more 
+  # details).  Here the default ConfiguredTask configurations are overridden
+  # by variations created by the variations! tag: 
   # 
   #   # [/path/to/app/config/template.yml]
   #   # one: ONE
@@ -113,47 +120,63 @@ module Tap
   # All together, these features make it easy to process a batch of inputs
   # using a batch of configurations -- all encapsulated in a workflow-ready
   # architecture.
+  #
+  # == Non-Task Tasks!
+  #
+  # The methods definining the essential behavior of a Task are encapsulated in 
+  # the Tap::Task::Base module.  Using this module, Non-Task classes can be defined, 
+  # and Non-Task objects extended to behave like Tasks; ie they can be enqued, run, 
+  # and incorporated into workflows.  
+  #
+  # See Tap::Task::Base for more details (esp for objects with a process method),
+  # as well as Tap::Support::Rake, which makes Rake tasks behave like Tap tasks.
+  #
   class Task < Monitor
-    write_inheritable_hash(:default_config, {})  
-    class_inheritable_hash(:default_config)  
+    write_inheritable_attribute(:configurations, Support::TaskConfiguration.new)  
+    class_inheritable_reader(:configurations)  
 
     write_inheritable_attribute(:iterative, true)  
     class_inheritable_reader(:iterative)
- 
+
+    write_inheritable_attribute(:source_files, [])  
+    class_inheritable_reader(:source_files)
+
     class << self
-  
-      # Sets the defualt configurations for the task class.  Configurations
-      # are inherited, but can be overridden or added to in subclasses.  Use
-      # options (:make_accessors, :make_readers, :make_writers) to create 
-      # accessors for configurations.
-      #
-      #   class SomeTask < Tap::Task
-      #     set_default_config({
-      #       :key => 'value'},
-      #       :make_accessors => true)
-      #   end
-      #
-      #   t = SomeTask.new
-      #   t.key               # => 'value'
-      #   t.key = 'another'
-      #   t.config            # => {:key => 'another'}
+      
+      # Currently an experimental way of identifying source files for TDoc
+      # documentation.
+      def source_file(arg) # :nodoc:
+        source_files << arg
+      end
+      
+      # Declares a configuration without any accessors.
       #
-      def set_default_config(hash={}, options={})
-        keys = hash.keys
-        self.write_inheritable_hash(:default_config, hash) 
-       
-        self.config_accessor(*keys) if options[:make_accessors]
-        self.config_reader(*keys) if options[:make_readers]
-        self.config_writer(*keys) if options[:make_writers]
+      # With no keys specified, sets config to make no accessors 
+      # for each new configuration.
+      def declare_config(*keys)
+        if keys.empty?
+          self.config_mode = :none 
+        else 
+          keys.each do |key|
+            configurations.declare(key, self)
+          end
+        end
       end
       
       # Creates a configuration writer for the input keys.  Works like
       # attr_writer, except the value is written to config, rather than
-      # a local variable.
+      # a local variable.  In addition, the config will be validated
+      # using validate_config upon setting the value.
+      #
+      # With no keys specified, sets config to create config_writer 
+      # for each new configuration.
       def config_writer(*keys)
-        keys.each do |key|
-          define_method("#{key}=") do |value|
-            config[key] = value
+        if keys.empty?
+          self.config_mode = :config_writer 
+        else 
+          keys.each do |key|
+            configurations.declare(key, self)
+            define_config_writer(key)
           end
         end
       end
@@ -161,10 +184,16 @@ module Tap
       # Creates a configuration reader for the input keys.  Works like
       # attr_reader, except the value is read from config, rather than
       # a local variable.
+      #
+      # With no keys specified, sets config to create a config_reader 
+      # for each new configuration.
       def config_reader(*keys)
-        keys.each do |key|
-          define_method(key) do
-            config[key]
+        if keys.empty?
+          self.config_mode = :config_reader 
+        else 
+          keys.each do |key|
+            configurations.declare(key, self)
+            define_config_reader(key)
           end
         end
       end
@@ -172,27 +201,248 @@ module Tap
       # Creates configuration accessors for the input keys.  Works like
       # attr_accessor, except the value is read from and written to config, 
       # rather than a local variable.
+      #
+      # With no keys specified, sets config to create a config_accessor 
+      # for each new configuration.
       def config_accessor(*keys)
-        config_writer(*keys)
-        config_reader(*keys)
+        if keys.empty?
+          self.config_mode = :config_accessor 
+        else 
+          keys.each do |key|
+            configurations.declare(key, self)
+            define_config_reader(key)
+            define_config_writer(key)
+          end
+        end
+      end
+      
+      # Sets a class configuration.  Configurations are inherited, but can 
+      # be overridden or added in subclasses. Accessors are created by 
+      # default, but this behavior can be modified by use of the other
+      # config methods.  
+      #
+      #   class SampleTask < Tap::Task
+      #     config :key, 'value'
+      #     
+      #     config_reader
+      #     config :reader_only
+      #   end
+      #
+      #   t = SampleTask.new
+      #   t.respond_to?(:reader_only)         # => true
+      #   t.respond_to?(:reader_only=)        # => false
+      #
+      #   t.config               # => {:key => 'value', :reader_only => nil} 
+      #   t.key                  # => 'value'
+      #   t.key = 'another'
+      #   t.config               # => {:key => 'another', :reader_only => nil}
+      def config(key, value=nil, attributes={})
+        declare_config(key)
+        configurations.set(key, value, attributes)
+        
+        case config_mode
+        when nil, :config_accessor then config_accessor(key)
+        when :config_writer then config_writer(key)
+        when :config_reader then config_reader(key)
+        end
       end
       
       # Sets the default iteration behavior for a task class 
-      # to iterate inputs during task execution.
+      # to iterate inputs during task execution.  As a result,
+      # tasks will NOT execute without inputs.
       def iterate
         self.write_inheritable_attribute(:iterative, true)  
       end
     
       # Sets the default iteration behavior for a task class 
-      # to not iterate inputs during task execution.
+      # to not iterate inputs during task execution.  As a result,
+      # tasks can execute without inputs.
       def do_not_iterate
         self.write_inheritable_attribute(:iterative, false)  
       end
+
+      protected
+      
+      attr_accessor :config_mode
+      
+      private
+      
+      def define_config_reader(key) # :nodoc:
+        define_method(key) do
+          config[key]
+        end
+      end
+      
+      def define_config_writer(key) # :nodoc:
+        define_method("#{key}=") do |value|
+          config[key] = value
+          validate_config(key, value)
+          value
+        end
+      end
     end
-   
-    attr_reader :name, :config, :config_template, :app, :batch, :condition_block, :task_block, :on_complete_block, :results
-    attr_accessor :multithread, :iterate
     
+    # Tap::Task::Base encapsulates the methods definining the essential 
+    # behavior of a Task.  
+    module Base
+      attr_reader :app, :batch, :condition_block, :on_complete_block, :results
+      attr_accessor :multithread, :iterate
+    
+      # Creates a new Task with the specified attributes.
+      def self.extended(base)
+        base.extend MonitorMixin
+      
+        base.instance_variable_set("@app", App.instance) if base.app.nil?
+        base.instance_variable_set("@batch", [base]) if base.batch.nil?
+        base.instance_variable_set("@results", []) if base.results.nil?
+        base.instance_variable_set("@condition_block", nil) if base.condition_block.nil?
+        base.instance_variable_set("@on_complete_block", nil) if base.on_complete_block.nil?
+        base.instance_variable_set("@multithread", false) if base.multithread.nil?
+        base.instance_variable_set("@iterate", false) if base.iterate.nil?
+      end
+      
+      # Returns true if the batch size is greater than one 
+      # (the one being the current task itself).  
+      def batched?
+        batch.length > 1
+      end
+    
+      # Returns the index of the current task in batch.
+      def batch_index
+        batch.index(self)
+      end
+    
+      # Returns true if multithread is true.
+      #
+      # (NOTE -- multithreading is currently experimental!)
+      def multithread?
+        multithread
+      end
+    
+      # Returns true if iterate is true.  If iterate has not been set for the task, 
+      # then iterate? returns the value of the class iterative variable.
+      def iterate?
+        iterate
+      end
+        
+      # Sets a condition block for the task.  Raises an error if condition_block is 
+      # already set, unless override = true.
+      #
+      # Note that the block will recieve the audited_inputs of the task
+      # (see Audit for more information).
+      def condition(override=false, &block) # :yields: self, audited_inputs
+        raise "Condition for task already set: #{self}" unless condition_block.nil? || override
+        self.condition_block = block
+      end
+    
+      # Sets a block to execute when the task completes execution.  Raises an error 
+      # if on_complete_block is already set, unless override = true.
+      #
+      # Note that the block will recieve the results of the task, ie an array of 
+      # audited values and not values themselves (see Audit for more information).
+      def on_complete(override=false, &block) # :yields: results
+        raise "On complete for task already set: #{self}" unless on_complete_block.nil? || override
+        self.on_complete_block = block
+      end
+    
+      # Returns true if no condition block is specified, or the condition
+      # block evaluates to true with the given inputs.
+      def executable?(inputs)
+        condition_block ? condition_block.call(self, inputs) : true
+      end
+    
+      # Execute the actions associated with this task.  Returns an array of Audits; 
+      # one for each input value.  Raises an error if the task is not executable with 
+      # the inputs, and executes the on_complete block when finished.
+      #
+      # Execute is synchronized such that a task can only execute on one thread at a 
+      # time.
+      def execute(*inputs)
+        synchronize do
+          check_terminate
+          
+          audited_inputs = if iterate? 
+            Support::Audit.register(*inputs)
+          else
+            Support::Audit.merge(*inputs)
+          end
+          
+          self.results = nil
+          raise "Condition not met." unless executable?(audited_inputs)
+          
+          before_execute
+          begin
+            self.results = if iterate? 
+              audited_inputs.collect do |audited_input| 
+                on_execute(audited_input)
+              end
+            else
+              [on_execute(audited_inputs)]
+            end
+          rescue
+            on_execute_error($!)
+          end
+          after_execute
+        
+          on_complete_block.call(results) if on_complete_block
+          results
+        end
+      end
+      
+      # Raises a TerminateError if the application is in state
+      # TERMINATE, as a mechanism to gracefully terminate threads.
+      #--
+      # Only occurs if the task is multithreaded.  
+      #++
+      #
+      # check_terminate is called before each execution, but can 
+      # be manually called at any time to provide a breakpoint
+      # in long executions.
+      def check_terminate
+        #if multithread? && app.state == App::State::TERMINATE
+        if app.state == App::State::TERMINATE
+          raise App::TerminateError.new
+        end
+      end
+      
+      protected
+
+      attr_writer :app, :batch, :condition_block, :on_complete_block, :results
+      
+      # Hook to execute code before inputs are processed.
+      def before_execute() end
+    
+      # Hook for overridding the execution code that generates the 
+      # task results.  The input will be an audited version of the
+      # inputs array provided to execute, or, if iterate? is true, 
+      # an audited version of each individual input provided to 
+      # execute.  on_execute should return the audited_inputs.
+      #
+      # By default, on_execute sends the current value of the audited 
+      # input process and then records the output with self as the source.
+      #
+      # It is not advised to override the default on_execute unless 
+      # absolutely necessary.  Consider adjusting the behavior of
+      # process first.
+      def on_execute(audited_inputs) 
+        output = process(audited_inputs._current)
+        audited_inputs._record(self, output)
+      end
+      
+      # Hook to execute code after inputs are processed, but before the on_complete block.
+      def after_execute() end
+
+      # Hook to handle unhandled errors from processing inputs on a task level.  
+      # By default on_execute_error simply re-raises the unhandled error.
+      def on_execute_error(err)
+        raise err
+      end
+    end
+    
+    include Base
+   
+    attr_reader :name, :config, :config_template, :task_block
+
     # Creates a new Task with the specified attributes.
     def initialize(name=nil, config=nil, app=App.instance, &task_block)
       super() # required for Monitor
@@ -237,28 +487,6 @@ module Tap
       task
     end
     
-    # Returns true if the batch size is greater than one 
-    # (the one being the current task itself).  
-    def batched?
-      batch.length > 1
-    end
-    
-    # Returns the index of the current task in batch.
-    def batch_index
-      batch.index(self)
-    end
-    
-    # Returns true if multithread is true.
-    def multithread?
-      multithread
-    end
-    
-    # Returns true if iterate is true.  If iterate has not been set for the task, 
-    # then iterate? returns the value of the class iterative variable.
-    def iterate?
-      iterate
-    end
-        
     # Returns the path to the configuration file used to make the task configuration
     # templates.  By default this is the YAML file for the task name relative to the
     # application config directory:
@@ -275,77 +503,26 @@ module Tap
     end
 
     # Configures the task with the given configuration overrides.  A Task has three
-    # configuration sources: the class default_config, the config_template loaded from
-    # config_file, and the inputs to this method. Configurations from these 
+    # configuration sources: the default class configurations, the config_template 
+    # loaded from config_file, and the inputs to this method. Configurations from these 
     # sources are merged as:  default_config.merge(config_template).merge(overrides)
     #
     # Configurations are symbolized before they are merged.  If overrides is nil, then
-    # they will be treated as an empty hash.
+    # they will be treated as an empty hash.  All configurations are validated using
+    # validate_config.
     def config=(overrides)
       overrides = overrides.nil? ? {} : overrides.symbolize_keys 
-      @config = self.class.default_config.merge(config_template).merge(overrides)
-     
+      @config = self.class.configurations.default.merge(config_template).merge(overrides)
+      self.config.each_pair {|key, value| validate_config(key, value) }
       self.config
     end
     
-    # Sets a condition block for the task.  Raises an error if condition_block is 
-    # already set, unless override = true.
-    def condition(override=false, &block) # :yields: self, inputs
-      raise "Condition for task already set: #{name}" unless condition_block.nil? || override
-      self.condition_block = block
-    end
-    
-    # Sets a block to execute when the task completes execution.  Raises an error 
-    # if on_complete_block is already set, unless override = true.
-    def on_complete(override=false, &block) # :yields: results
-      raise "On complete for task already set: #{name}" unless on_complete_block.nil? || override
-      self.on_complete_block = block
-    end
-    
-    # Returns true if no condition block is specified, or the condition
-    # block evaluates to true with the given inputs.
-    def executable?(inputs)
-      condition_block ? condition_block.call(self, inputs) : true
-    end
-    
-    # Execute the actions associated with this task.  Returns an array of Audits; 
-    # one for each input value.  Raises an error if the task is not executable with 
-    # the inputs, and executes the on_complete block when finished.
-    #
-    # Execute is synchronized such that a task can only execute on one thread at a 
-    # time.
-    def execute(*inputs)
-      synchronize do
-        self.results = nil
-        raise "Condition not met." unless executable?(inputs)
-
-        inputs = case
-        when inputs.empty? then inputs
-        when iterate? then Support::Audit.register(*inputs)
-        else [Support::Audit.merge(*inputs)]
-        end
-        
-        before_execute
-        begin 
-          self.results = inputs.each do |input|
-            output = process( input._current )
-            input._record(self, output)
-          end
-        rescue
-          on_execute_error($!)
-        end
-        after_execute
-        
-        on_complete_block.call(results) if on_complete_block
-        results
-      end
-    end
-    
     # The method for processing inputs into outputs.  Override this method in
     # subclasses to provide class-specific process logic.  By default the 
     # input is passed to the task_block (provided during initialization).
     # Simply returns the input if no task_block is set.
     def process(input)
+      check_terminate
       task_block.nil? ? input : task_block.call(self, input)
     end
     
@@ -362,20 +539,12 @@ module Tap
     
     protected
 
-    attr_writer :name, :config_template, :app, :batch, :condition_block, :task_block, :on_complete_block, :results
-
-    # Hook to execute code before inputs are processed.
-    def before_execute() end
+    attr_writer :name, :config_template, :task_block
     
-    # Hook to execute code after inputs are processed, but before the on_complete block.
-    def after_execute() end
-
-    # Hook to handle unhandled errors from processing inputs on a task level.  
-    # By default on_execute_error simply re-raises the unhandled error.
-    def on_execute_error(err)
-      raise err
+    # Hook to validate configurations.
+    def validate_config(key, value)
     end
-
+    
     # Hook to provide class-specific processing of the config_file template
     # (ie app.config_template(config_file)).  By default the template is 
     # processed using Support::Templater.make_templates.
@@ -388,61 +557,4 @@ module Tap
     end
 
   end
-end
-
-class Hold # :nodoc:
-  attr_accessor :help_doc
-  
-  def register_doc(*section_headers)
-    section_headers << "Command Line Usage" if section_headers.empty?
-    self.help_doc = [caller.first.sub(/:\d+$/, ''), section_headers.collect {|section| section.strip}]
-  end
-  
-  def help
-    return "No help available for #{self}" unless help_doc
-    
-    lines = []
-    help_file, sections = help_doc
-    File.open(help_file) do |file|
-      in_section = nil
-      while line = file.gets
-        next unless line =~ /^#(.*)/
-        line = $1.strip
-        
-        if line =~ /^=+(.*)/
-          line = $1.strip
-          in_section = sections.delete(line) 
-          line += ":"
-        else
-          line = "  " + line
-        end
-        
-        lines << line if in_section
-      end
-    end
-  
-    return lines.join("\n")
-  end
-    
-    # Queues self to app with the input.
-    #def <<(input)
-      #queue(input)
-    #end
-    
-   # Ticks the monitor by printing '.', or the specified message.
-   # Only ticks if called from within a monitor block.
-   #def tick_monitor(action=nil, msg="", level=Logger::INFO)
-    # app.tick_monitor(action, msg, level)
-   #end
-   
-   # Prints the hash of statistics if called within a monitor block.
-   #def monitor_stats(hash)
-    # app.monitor_stats(hash)
-   #end
-   
-   # Times the execution of the block and enables the various monitor
-   # methods (tick_monitor, monitor_stats).  
-  # def monitor(action="beginning", msg="", &block)
-   #  app.monitor(action,msg, &block)
-   #end
 end