tap 0.12.4 → 0.17.0

data/doc/Examples/Command Line

@@ -0,0 +1,36 @@
+= Command Line Examples
+
+== Basic Input/Output
+
+=== Read data from $stdin
+  # [goodnight.txt]
+  # goodnight moon
+  
+  % tap run -- load --: dump < goodnight.txt
+  goodnight moon
+
+=== Pipe data from $stdin
+  % echo goodnight moon | tap run -- load --: dump
+  goodnight moon
+
+=== Load data from argument
+  % tap run -- load 'goodnight moon' --: dump
+  goodnight moon
+
+=== Dump data to $stdout
+  % tap run -- load 'goodnight moon' --: dump > goodnight.txt
+  % more goodnight.txt
+  goodnight moon
+
+=== Pipe data via $stdout
+  % tap run -- load 'goodnight moon' --: dump | more
+  goodnight moon
+
+== Setup
+
+=== Use an ENV variable to setup the Tap::Env
+
+  % TAP_GEMS= tap run -T
+  % TAP_GEMS=:all tap run -T
+  % TAP_GEMS=:latest tap run -T
+  % TAP_GEMS="[rap, tap-tasks]" tap run -T

data/doc/Examples/Workflow

@@ -0,0 +1,40 @@
+= Workflow Examples
+
+=== Sequence
+
+A simple sequence using the --: syntax that joins the previous to next task.
+
+  % tap run -- load 'goodnight moon' --: dump
+  goodnight moon
+  
+=== Sequence (canonical)
+
+The more canonical way of specifying a sequence.
+
+  % tap run -- load 'goodnight moon' -- dump --[0][1]
+  goodnight moon
+
+=== Fork
+
+Multiple outputs for a single input.
+
+  % tap run -- load 'goodnight moon' -- dump -- dump --[0][1,2]
+  goodnight moon
+  goodnight moon
+  
+=== Merge
+
+Multiple inputs for a single output; note that as seen in the output, dump
+receives the inputs in serial, whenever they happen to be ready.
+
+  % tap run -- load goodnight -- load moon -- dump --[0,1][2]
+  goodnight
+  moon
+
+=== Synchronized Merge
+
+Similar to a merge, but the results of each input are collected into an array
+before being passed to dump. The printout is: ['goodnight', 'moon'].to_s
+
+  % tap run -- load goodnight -- load moon -- dump --[0,1][2].sync
+  goodnightmoon

data/lib/tap/app.rb

@@ -1,207 +1,181 @@
 require 'logger'
-
-require 'tap/support/aggregator'
-require 'tap/support/dependencies'
-require 'tap/support/executable_queue'
-require 'tap/task'
+require 'configurable'
+require 'tap/app/node'
+require 'tap/app/state'
+require 'tap/app/stack'
+require 'tap/app/queue'
 
 module Tap
   
-  # App coordinates the setup and running of tasks, and provides an interface 
-  # to the application directory structure. All tasks have an app (by default
-  # App.instance) through which they access application-wide resources like
-  # the logger, executable queue, and dependencies.
-  #
-  # === Running Tasks
+  # App coordinates the setup and execution of workflows.
   #
-  # Tasks may be enqued and run by an App:
+  # === Workflows
   #
-  #   app = App.instance
+  # Workflows are composed of nodes and joins such as instances of Tap::Task
+  # and Tap::Join.  The actual workflow exists between nodes; each node can
+  # specify a join to receive it's output and enque or execute other nodes.
+  # When a node does not have a join, apps allow the specification of a
+  # default join to, for instance, aggregate results.
   #
-  #   t = Task.intern {|task, *inputs| inputs }
-  #   t.enq('a', 'b', 'c')
-  #   t.enq(1)
-  #   t.enq(2)
-  #   t.enq(3)
-  #
-  #   app.run
-  #   app.results(t)                 # => [['a', 'b', 'c'], [1], [2], [3]]
+  # Any object satisfying the correct API[link:files/doc/API.html] can be used
+  # as a node or join.  Apps have helpers to make nodes out of blocks.
   #
-  # By default apps simply run tasks and collect the results.  To construct
-  # a workflow, set an on_complete block to receive the audited result and
-  # enque or execute the next series of tasks.  Here is a simple sequence:
+  #   app = Tap::App.new
+  #   n = app.node {|*inputs| inputs }
+  #   app.enq(n, 'a', 'b', 'c')
+  #   app.enq(n, 1)
+  #   app.enq(n, 2)
+  #   app.enq(n, 3)
   #
-  #   t0 = Task.intern {|task| "0" }
-  #   t1 = Task.intern {|task, input| "#{input}:1" }
-  #   t2 = Task.intern {|task, input| "#{input}:2"}
+  #   results = []
+  #   app.on_complete {|result| results << result }
   #
-  #   t0.on_complete {|_result| t1.enq(_result) }
-  #   t1.on_complete {|_result| t2.enq(_result) }
-  #   
-  #   t0.enq
   #   app.run
-  #   app.results(t0, t1)            # => []
-  #   app.results(t2)                # => ["0:1:2"]
+  #   results                        # => [['a', 'b', 'c'], [1], [2], [3]]
   #
-  # Apps may be assigned an on_complete block as well; the app on_complete
-  # block is called when a task has no on_complete block set.  If neither the
-  # task nor the app has an on_complete block, the app stores the audit in
-  # app.aggregator and makes it available through app.results.  Note how after
-  # the sequence, the t0 and t1 results are not in the app (they were handled
-  # by the on_complete block).
+  # To construct a workflow, set joins for individual nodes.  Here is a simple
+  # sequence:
   #
-  # Tracking how inputs evolve through a workflow can be onerous.  To help,
-  # Tap audits changes to the inputs.  Audit values are by convention prefixed
-  # by an underscore; all on_complete block receive the audited values and not
-  # the actual result of a task.  Aggregated audits are available through
-  # app._results.
+  #   n0 = app.node { "a" }
+  #   n1 = app.node {|input| "#{input}.b" }
+  #   n2 = app.node {|input| "#{input}.c"}
   #
-  #   t2.enq("a")
-  #   t1.enq("b")
+  #   n0.on_complete {|result| app.execute(n1, result) }
+  #   n1.on_complete {|result| app.execute(n2, result) }
+  #   app.enq(n0)
+  #
+  #   results.clear
   #   app.run
-  #   app.results(t2)                # => ["0:1:2", "a:2", "b:1:2"]
-  # 
-  #   t0.name = "zero"
-  #   t1.name = "one"
-  #   t2.name = "two"
+  #   results                        # => ["a.b.c"]
   #
-  #   trails = app._results(t2).collect do |_result|
-  #     _result.dump
-  #   end
+  # Tasks have helpers to simplify the manual constructon of workflows, but
+  # even with these methods large workflows are cumbersome to build.  More
+  # typically, a Tap::Schema is used in such cases.
   #
-  #   "\n" + trails.join("\n")
-  #   # => %q{
-  #   # o-[zero] "0"
-  #   # o-[one] "0:1"
-  #   # o-[two] "0:1:2"
-  #   # 
-  #   # o-[] "a"
-  #   # o-[two] "a:2"
-  #   # 
-  #   # o-[] "b"
-  #   # o-[one] "b:1"
-  #   # o-[two] "b:1:2"
-  #   # }
+  # === Middleware
   #
-  # See Audit for more details.
+  # Apps allow middleware to wrap the execution of each node.  This can be
+  # particularly useful to track the progress of a workflow.  Middleware is
+  # initialized with the application stack and uses the call method to
+  # wrap the execution of the stack.
   #
-  # === Dependencies
+  # Using middleware, an auditor looks like this:
   #
-  # Tasks allow the construction of dependency-based workflows.  A dependent
-  # task only executes after its dependencies have been resolved.
+  #   class AuditMiddleware
+  #     attr_reader :stack, :audit
   #
-  #   runlist = []
-  #   t0 = Task.intern {|task| runlist << task }
-  #   t1 = Task.intern {|task| runlist << task }
+  #     def initialize(stack)
+  #       @stack = stack
+  #       @audit = []
+  #     end
   #
-  #   t0.depends_on(t1)
-  #   t0.enq
+  #     def call(node, inputs=[])
+  #       audit << node
+  #       stack.call(node, inputs)
+  #     end
+  #   end
   #
-  #   app.run
-  #   runlist                        # => [t1, t0]
+  #   auditor = app.use AuditMiddleware
   #
-  # Once a dependency is resolved, it will not execute again:
+  #   app.enq(n0)
+  #   app.enq(n2, "x")
+  #   app.enq(n1, "y")
   #
-  #   t0.enq
+  #   results.clear
   #   app.run
-  #   runlist                        # => [t1, t0, t0]
+  #   results                        # => ["a.b.c", "x.c", "y.b.c"]
+  #   auditor.audit                  
+  #   # => [
+  #   # n0, n1, n2, 
+  #   # n2,
+  #   # n1, n2
+  #   # ]
+  # 
+  # Middleware can be nested with multiple calls to use.
   #
-  # === Executables
+  # === Dependencies
   #
-  # App can enque and run any Executable object. Arbitrary methods may be
-  # made into Executables using Object#_method.
+  # Nodes allow the construction of dependency-based workflows.  A node only
+  # executes after its dependencies have been resolved (ie executed).
   #
-  #   array = []
+  #   runlist = []
+  #   n0 = app.node { runlist << 0 }
+  #   n1 = app.node { runlist << 1 }
   #
-  #   m = array._method(:push)
-  #   m.enq(1)
-  #   m.enq(2)
-  #   m.enq(3)
+  #   n0.depends_on(n1)
+  #   app.enq(n0)
   #
-  #   array.empty?                   # => true
   #   app.run
-  #   array                          # => [1, 2, 3]
+  #   runlist                        # => [1, 0]
   #
-  class App < Monitor
+  # Dependencies are resolved <em>every time</em> a node executes; individual
+  # dependencies can implement single-execution if desired.  Dependencies are
+  # not resolved with arguments, ie dependency nodes must be able to execute
+  # without inputs.
+  #
+  class App
     class << self
       # Sets the current app instance
       attr_writer :instance
       
       # Returns the current instance of App.  If no instance has been set,
-      # then instance initializes a new App with the default configuration. 
-      def instance
-        @instance ||= App.new
+      # then instance initializes a new App with the default configuration.
+      #
+      # Instance is used to initialize tasks when no app is specified.  Aside
+      # from that, there is nothing magical about instance.
+      def instance(auto_initialize=true)
+        @instance ||= (auto_initialize ? new : nil)
       end
     end
     
     include Configurable
+    include MonitorMixin
     
-    # The application logger
-    attr_reader :logger
-    
-    # The application queue
-    attr_reader :queue
+    # The default App logger writes to $stderr at level INFO.
+    DEFAULT_LOGGER = Logger.new($stderr)
+    DEFAULT_LOGGER.level = Logger::INFO
+    DEFAULT_LOGGER.formatter = lambda do |severity, time, progname, msg|
+      "  %s[%s] %18s %s\n" % [severity[0,1], time.strftime('%H:%M:%S') , progname || '--' , msg]
+    end
     
     # The state of the application (see App::State)
     attr_reader :state
     
-    # A Tap::Support::Aggregator that collects the results of 
-    # methods that have no on_complete block
-    attr_reader :aggregator
+    # The application call stack for executing nodes
+    attr_reader :stack
     
-    # A Tap::Support::Dependencies to track dependencies.
-    attr_reader :dependencies
+    # The application queue
+    attr_reader :queue
     
-    # The block called when an executable completes and has no
-    # on_complete_block set.  An on_complete_block effectively
-    # takes the place of aggregation (ie when set, no results
-    # will be collected by self).
-    attr_reader :on_complete_block
+    # A cache of application-specific data.  Internally used to store class
+    # instances of tasks.  Not recommended for casual use.
+    attr_reader :cache
+    
+    # The default joins for nodes that have no joins set
+    attr_accessor :default_joins
+    
+    # The application logger
+    attr_reader :logger
     
-    config :audit, true, &c.switch                # Signal auditing
     config :debug, false, &c.flag                 # Flag debugging
     config :force, false, &c.flag                 # Force execution at checkpoints
     config :quiet, false, &c.flag                 # Suppress logging
     config :verbose, false, &c.flag               # Enables extra logging (overrides quiet)
     
-    # The constants defining the possible App states.  
-    module State
-      READY = 0
-      RUN = 1
-      STOP = 2
-      TERMINATE = 3
-      
-      module_function
-      
-      # Returns a string corresponding to the input state value.  
-      # Returns nil for unknown states.
-      #
-      #   State.state_str(0)        # => 'READY'
-      #   State.state_str(12)       # => nil
-      def state_str(state)
-        constants.inject(nil) {|str, s| const_get(s) == state ? s.to_s : str}
-      end
-    end
-    
     # Creates a new App with the given configuration.  
-    def initialize(config={}, logger=DEFAULT_LOGGER, &block)
+    def initialize(config={}, options={}, &block)
       super() # monitor
       
       @state = State::READY
-      @queue = Support::ExecutableQueue.new
-      @aggregator = Support::Aggregator.new
-      @dependencies = Support::Dependencies.new
-      @on_complete_block = block
+      @stack = options[:stack] || Stack.new
+      @queue = options[:queue] || Queue.new
+      @cache = options[:cache] || {}
+      @trace = []
+      @default_joins = []
+      on_complete(&block)
       
       initialize_config(config)
-      self.logger = logger
-    end
-    
-    # The default App logger writes to $stderr at level INFO.
-    DEFAULT_LOGGER = Logger.new($stderr)
-    DEFAULT_LOGGER.level = Logger::INFO
-    DEFAULT_LOGGER.formatter = lambda do |severity, time, progname, msg|
-      "  %s[%s] %18s %s\n" % [severity[0,1], time.strftime('%H:%M:%S') , progname || '--' , msg]
+      self.logger = options[:logger] || DEFAULT_LOGGER
     end
     
     # True if debug or the global variable $DEBUG is true.
@@ -225,19 +199,111 @@ module Tap
       logger.add(level, msg, action.to_s) if !quiet || verbose
     end
     
-    # Sequentially calls execute with the (executable, inputs) pairs in
-    # queue; run continues until the queue is empty and then returns self.
+    # Returns a new node that executes block on call.
+    def node(&block) # :yields: *inputs
+      Node.intern(&block)
+    end
+    
+    # Enques the node with the inputs.  Returns the node.
+    def enq(node, *inputs)
+      queue.enq(node, inputs)
+      node
+    end
+    
+    # Generates a node from the block and enques. Returns the new node.
+    def bq(*inputs, &block) # :yields: *inputs
+      node = self.node(&block)
+      queue.enq(node, inputs)
+      node
+    end
+    
+    # Adds the specified middleware to the stack.
+    def use(middleware)
+      @stack = middleware.new(@stack)
+    end
+    
+    # Clears the cache, the queue, and resets the stack so that no middleware
+    # is used.  Reset raises an error unless state == State::READY.
+    def reset
+      synchronize do
+        unless state == State::READY
+          raise "cannot reset unless READY"
+        end
+        
+        @stack = Stack.new
+        cache.clear
+        queue.clear
+      end
+    end
+    
+    # Dispatches each dependency of node.  A block can be given to do something
+    # else with the nodes (ex: reset single-execution dependencies).  Resolve
+    # will recursively yield dependencies if specified.
+    #
+    # Resolve raises an error for circular dependencies.
+    def resolve(node, recursive=false, &block)
+      node.dependencies.each do |dependency|
+        if @trace.include?(dependency)
+          @trace.push dependency
+          raise DependencyError.new(@trace)
+        end
+
+        # mark the results at the index to prevent
+        # infinite loops with circular dependencies
+        @trace.push dependency
+        
+        if recursive
+          resolve(dependency, recursive, &block)
+        end
+        
+        if block_given?
+          yield(dependency)
+        else
+          dispatch(dependency)
+        end
+        
+        @trace.pop
+      end
+    end
+    
+    # Dispatches node to the application stack with the inputs.
+    def execute(node, *inputs)
+      dispatch(node, inputs)
+    end
+    
+    # Dispatch sends the node into the application stack with the inputs.
+    # Dispatch does the following in order:
+    #
+    # - resolve node dependencies using resolve_dependencies
+    # - call stack with the node and inputs
+    # - call the node joins, if set, or the default_joins with the results
+    #
+    # Dispatch returns the node result.
+    def dispatch(node, inputs=[])
+      resolve(node)
+      result = stack.call(node, inputs)
+      
+      joins = node.joins.empty? ? default_joins : node.joins
+      joins.each do |join|
+        join.call(result)
+      end
+      result
+    end
+    
+    # Sequentially dispatches each enqued (node, inputs) pair to the
+    # application stack.  A run continues until the queue is empty.  Returns
+    # self.
     #
     # ==== Run State
     #
-    # Run checks the state of self before executing a method. If state
+    # Run checks the state of self before dispatching a node.  If the state
     # changes from State::RUN, the following behaviors result:
     # 
-    # State::STOP:: No more executables will be executed; the current
-    #               executable will continute to completion.
-    # State::TERMINATE:: No more executables will be executed and the
-    #                    currently running executable will be
-    #                    discontinued as described in terminate.
+    # State::STOP:: No more nodes will be dispatched; the current node will
+    #               continute to completion.
+    # State::TERMINATE:: No more nodes will be dispatched and the currently
+    #                    running node will be discontinued as described in
+    #                    terminate.
     #
     # Calls to run when the state is not State::READY do nothing and
     # return immediately.
@@ -250,8 +316,7 @@ module Tap
       # TODO: log starting run
       begin
         until queue.empty? || state != State::RUN
-          executable, inputs = queue.deq
-          executable._execute(*inputs)
+          dispatch(*queue.deq)
         end
       rescue(TerminateError)
         # gracefully fail for termination errors
@@ -267,9 +332,9 @@ module Tap
       self
     end
     
-    # Signals a running application to stop executing tasks in the 
-    # queue by setting state = State::STOP.  The task currently 
-    # executing will continue uninterrupted to completion.
+    # Signals a running app to stop dispatching nodes to the application stack
+    # by setting state = State::STOP.  The node currently in the stack will
+    # will continue to completion.
     #
     # Does nothing unless state is State::RUN.
     def stop
@@ -278,10 +343,14 @@ module Tap
     end
 
     # Signals a running application to terminate execution by setting 
-    # state = State::TERMINATE.  In this state, an executing task 
-    # will then raise a TerminateError upon check_terminate, thus 
-    # allowing the invocation of task-specific termination, perhaps 
-    # performing rollbacks. (see Tap::Support::Executable#check_terminate).
+    # state = State::TERMINATE.  In this state, calls to check_terminate
+    # will raise a TerminateError.  Run considers TerminateErrors a normal
+    # exit and rescues them quietly.
+    #
+    # Nodes can set breakpoints that call check_terminate to invoke
+    # node-specific termination.  If a node never calls check_terminate, then
+    # it will continue to completion and terminate is functionally the same
+    # as stop.
     #
     # Does nothing if state == State::READY.
     def terminate
@@ -289,76 +358,86 @@ module Tap
       self
     end
     
+    # Raises a TerminateError if state == State::TERMINATE.  Nodes should call
+    # check_terminate to provide breakpoints in long-running processes.
+    def check_terminate
+      if state == App::State::TERMINATE
+        raise App::TerminateError.new
+      end
+    end
+    
     # Returns an information string for the App.  
     #
-    #   App.instance.info   # => 'state: 0 (READY) queue: 0 results: 0'
+    #   App.instance.info   # => 'state: 0 (READY) queue: 0'
     #
     def info
-      "state: #{state} (#{State.state_str(state)}) queue: #{queue.size} results: #{aggregator.size}"
+      "state: #{state} (#{State.state_str(state)}) queue: #{queue.size}"
     end
     
-    # Enques the task (or Executable) with the inputs.  Raises an error if the
-    # input is not an Executable, or is not assigned to self.  Returns task.
-    def enq(task, *inputs)
-      unless task.kind_of?(Support::Executable)
-        raise ArgumentError, "not an Executable: #{task.inspect}"
-      end
-      
-      unless task.app == self
-        raise ArgumentError, "not assigned to enqueing app: #{task.inspect}"
-      end
-      
-      queue.enq(task, inputs)
-      task
-    end
-
-    # Method enque.  Enques the specified method from object with the inputs.
-    # Returns the enqued method.
-    def mq(object, method_name, *inputs)
-      m = object._method(method_name)
-      enq(m, *inputs)
-    end
-        
-    # Returns all aggregated, audited results for the specified tasks.  
-    # Results are joined into a single array.  Arrays of tasks are 
-    # allowed as inputs. See results.
-    def _results(*tasks)
-      aggregator.retrieve_all(*tasks.flatten)
-    end
-    
-    # Returns all aggregated results for the specified tasks.  Results are
-    # joined into a single array.  Arrays of tasks are allowed as inputs.    
+    # Dumps self to the target as YAML.
     #
-    #   t0 = Task.intern  {|task, input| "#{input}.0" }
-    #   t1 = Task.intern  {|task, input| "#{input}.1" }
+    # ==== Notes
     #
-    #   t0.enq(0)
-    #   t1.enq(1)
+    # Platforms that use {Syck}[http://whytheluckystiff.net/syck/] (ex MRI)
+    # require a fix because Syck misformats certain dumps, such that they
+    # cannot be reloaded (even by Syck).  Specifically:
     #
-    #   app.run
-    #   app.results(t1, t0)           # => ["1.1", "0.0"]
+    #   &id001 !ruby/object:Tap::Task ?
     #
-    def results(*tasks)
-      _results(tasks).collect {|_result| _result.value }
-    end
-    
-    # Sets a block to receive the results tasks with no on_complete_block
-    # set.  Raises an error if an on_complete_block is already set.
-    # Override the existing on_complete_block by specifying override = true.
+    # should be:
+    #
+    #   ? &id001 !ruby/object:Tap::Task
     #
-    # Note: the block recieves an audited result and not the result
-    # itself (see Audit for more information).
-    def on_complete(override=false, &block) # :yields: _result
-      unless on_complete_block == nil || override
-        raise "on_complete_block already set: #{self}" 
+    # Dump fixes this error and, in addition, removes Thread and Proc dumps
+    # because they can't be allocated on load.
+    def dump(target=$stdout, options={})
+      synchronize do
+        options = {
+          :date_format => '%Y-%m-%d %H:%M:%S',
+          :date => true,
+          :info => true
+        }.merge(options)
+        
+        # print basic headers
+        target.puts "# date: #{Time.now.strftime(options[:date_format])}" if options[:date]
+        target.puts "# info: #{info}" if options[:info]
+        
+        # # print load paths and requires
+        # target.puts "# load paths"
+        # target.puts $:.to_yaml
+        # 
+        # target.puts "# requires"
+        # target.puts $".to_yaml
+        
+        # dump yaml, fixing as necessary
+        yaml = YAML.dump(self)
+        yaml.gsub!(/\&(.*!ruby\/object:.*?)\s*\?/) {"? &#{$1} " } if YAML.const_defined?(:Syck)
+        yaml.gsub!(/!ruby\/object:(Thread|Proc) \{\}/, '')
+        target << yaml
       end
-      @on_complete_block = block
+      
+      target
+    end
+    
+    # Sets the block to receive the audited result of nodes with no join
+    # (ie the block is set as default_join).
+    def on_complete(&block) # :yields: _result
+      self.default_joins << block if block
       self
     end
     
-    # TerminateErrors are raised to kill executing tasks when terminate is 
+    protected
+    
+    # TerminateErrors are raised to kill executing nodes when terminate is 
     # called on an running App.  They are handled by the run rescue code.
-    class TerminateError < RuntimeError 
+    class TerminateError < RuntimeError
+    end
+    
+    # Raised when Tap::App#resolve detects a circular dependency.
+    class DependencyError < StandardError
+      def initialize(trace)
+        super "circular dependency: [#{trace.join(', ')}]"
+      end
     end
   end
 end