tap 0.10.1 → 0.11.0

data/lib/tap/app.rb

@@ -1,103 +1,96 @@
 require 'logger'
-require 'tap/support/run_error'
 require 'tap/support/aggregator'
+require 'tap/support/dependencies'
 require 'tap/support/executable_queue'
 
 module Tap
   
   # App coordinates the setup and running of tasks, and provides an interface 
-  # to the application directory structure.  App is convenient for use within 
-  # scripts and, with Env, provides the basis for the 'tap' command line 
-  # application.  
+  # to the application directory structure. All tasks have an App (by default
+  # App.instance) through which tasks access access application-wide resources 
+  # like the logger, executable queue, aggregator, and dependencies.
   #
   # === Running Tasks
   #
-  # All tasks have an App (by default App.instance) through which tasks access
-  # access application-wide resources like the logger.  Additionally, task
-  # enque command are forwarded to App#enq:
+  # Task enque command are forwarded to App#enq:
   #
-  #   t1 = Task.new {|task, input| input += 1 }
-  #   t1.enq(0)
-  #   app.enq(t1, 1)
+  #   t0 = Task.intern {|task, input| "#{input}.0" }
+  #   t0.enq('a')
+  #   app.enq(t0, 'b')
   #
   #   app.run
-  #   app.results(t1)                # => [1, 2]
+  #   app.results(t0)                # => ['a.0', 'b.0']
   #
-  # When a task completes, the results will either be passed to the task
-  # <tt>on_complete</tt> block (if set) or be collected into an Aggregator;
-  # aggregated results may be accessed per-task, as shown above.  Task 
-  # <tt>on_complete</tt> blocks typically enque other tasks, allowing the
-  # construction of workflows:
+  # When a task completes, the results will be passed to the task on_complete
+  # block, if set, or be collected into an Aggregator (aggregated results may 
+  # be accessed per-task, as shown above); on_complete blocks typically 
+  # execute or enque other tasks, allowing the construction of imperative
+  # workflows:
   #
   #   # clear the previous results
   #   app.aggregator.clear
   #
-  #   t2 = Task.new {|task, input| input += 10 }
-  #   t1.on_complete {|_result| t2.enq(_result) }
-  #
-  #   t1.enq 0
-  #   t1.enq 10
+  #   t1 = Task.intern {|task, input| "#{input}.1" }
+  #   t0.on_complete {|_result| t1.enq(_result) }
+  #   t0.enq 'c'
   #
   #   app.run
-  #   app.results(t1)                # => []
-  #   app.results(t2)                # => [11, 21]
+  #   app.results(t0)                # => []
+  #   app.results(t1)                # => ['c.0.1']
   #
-  # Here t1 has no results because the on_complete block passed them to t2 in 
+  # Here t0 has no results because the on_complete block passed them to t1 in 
   # a simple sequence.
   #
-  # ==== Batching
+  # === Dependencies
   #
-  # Tasks can be batched, allowing the same input to be enqued to multiple 
-  # tasks at once.
+  # Tasks allow the construction of dependency-based workflows such that a 
+  # dependent task only executes after its dependencies have been resolved.
   #
-  #   t1 = Task.new  {|task, input| input += 1 }
-  #   t2 = Task.new  {|task, input| input += 10 }
-  #   Task.batch(t1, t2)             # => [t1, t2]
+  #   runlist = []
+  #   t0 = Task.intern {|task| runlist << task }
+  #   t1 = Task.intern {|task| runlist << task }
   #
-  #   t1.enq 0
+  #   t0.depends_on(t1)
+  #   t0.enq
   #
   #   app.run
-  #   app.results(t1)                # => [1]
-  #   app.results(t2)                # => [10]
+  #   runlist                        # => [t1, t0]
   #
-  # ==== Multithreading
+  # Once a dependency is resolved, it will not execute again:
   #
-  # App supports multithreading; multithreaded tasks execute cosynchronously, 
-  # each on their own thread.  
+  #   t0.enq
+  #   app.run
+  #   runlist                        # => [t1, t0, t0]
   #
-  #   lock = Mutex.new
-  #   array = []
-  #   t1 = Task.new  {|task| lock.synchronize { array << Thread.current.object_id }; sleep 0.1 }
-  #   t2 = Task.new  {|task| lock.synchronize { array << Thread.current.object_id }; sleep 0.1 }
-  #   
-  #   t1.multithread = true
-  #   t1.enq
-  #   t2.multithread = true
-  #   t2.enq
+  # === Batching
   #
-  #   app.run
-  #   array.length                   # => 2
-  #   array[0] == array[1]           # => false
+  # Tasks can be batched, allowing the same input to be enqued to multiple 
+  # tasks at once.
+  #
+  #   t0 = Task.intern  {|task, input| "#{input}.0" }
+  #   t1 = Task.intern  {|task, input| "#{input}.1" }
   #
-  # Naturally, it is up to you to make sure each task is thread safe.  Note 
-  # that for the most part Tap::App is NOT thread safe; only run and 
-  # run-related methods (ready, stop, terminate, info) are synchronized.
-  # Methods enq and results act on thread-safe objects ExecutableQueue and 
-  # Aggregator, and should be ok to use from multiple threads.
+  #   t0.batch_with(t1)
+  #   t0.enq 'a'
+  #   t1.enq 'b'
   #
-  # ==== Executables
+  #   app.run
+  #   app.results(t0)                # => ['a.0', 'b.0']
+  #   app.results(t1)                # => ['a.1', 'b.1']
   #
-  # App can use any Executable object in place of a task. One way to initialize 
-  # an Executable for a method is to use the Object#_method defined by Tap.  The 
-  # result can be enqued and incorporated into workflows, but they cannot be 
-  # batched.
+  # === Executables
   #
-  # The mq (method enq) method generates and enques the method in one step.
+  # App can enque and run any Executable object. Arbitrary methods may be
+  # made into Executables using Object#_method. The mq (method enq) method
+  # generates and enques methods in one step.
   #
   #   array = []
+  #
+  #   # longhand
   #   m = array._method(:push)
-  #    
-  #   app.enq(m, 1)
+  #   m.enq(1)
+  #
+  #   # shorthand
   #   app.mq(array, :push, 2)
   #
   #   array.empty?                   # => true
@@ -106,32 +99,30 @@ module Tap
   #
   # === Auditing
   # 
-  # All results generated by executable methods are audited to track how a given  
-  # input evolves during a workflow.   
+  # All results are audited to track how a given input evolves during a workflow.
+  # To illustrate auditing, consider and addition workflow that ends in eights.
   #
-  # To illustrate auditing, consider a workflow that uses the 'add_one' method 
-  # to add one to an input until the result is 3, then adds five more with the 
-  # 'add_five' method.  The final result should always be 8.  
+  #   add_one  = Tap::Task.intern({}, 'add_one')  {|task, input| input += 1 }
+  #   add_five = Tap::Task.intern({}, 'add_five') {|task, input| input += 5 }
   #
-  #   t1 = Tap::Task.new {|task, input| input += 1 }
-  #   t1.name = "add_one"
-  #
-  #   t2 = Tap::Task.new {|task, input| input += 5 }
-  #   t2.name = "add_five"
-  #
-  #   t1.on_complete do |_result|
+  #   add_one.on_complete do |_result|
   #     # _result is the audit; use the _current method
   #     # to get the current value in the audit trail
+  #     current_value = _result._current
   #
-  #     _result._current < 3 ? t1.enq(_result) : t2.enq(_result)
+  #     if current_value < 3 
+  #       add_one.enq(_result)
+  #     else
+  #       add_five.enq(_result)
+  #     end
   #   end
   #   
-  #   t1.enq(0)
-  #   t1.enq(1)
-  #   t1.enq(2)
+  #   add_one.enq(0)
+  #   add_one.enq(1)
+  #   add_one.enq(2)
   #
   #   app.run
-  #   app.results(t2)                # => [8,8,8]
+  #   app.results(add_five)          # => [8,8,8]
   #
   # Although the results are indistinguishable, each achieved the final value
   # through a different series of tasks. With auditing you can see how each 
@@ -139,7 +130,7 @@ module Tap
   #
   #   # app.results returns the actual result values
   #   # app._results returns the audits for these values
-  #   app._results(t2).each do |_result|
+  #   app._results(add_five).each do |_result|
   #     puts "How #{_result._original} became #{_result._current}:"
   #     puts _result._to_s
   #     puts
@@ -167,8 +158,6 @@ module Tap
   #
   # See Tap::Support::Audit for more details.
   class App < Root
-    include MonitorMixin
-
     class << self
       # Sets the current app instance
       attr_writer :instance
@@ -190,14 +179,17 @@ module Tap
     attr_reader :state
     
     # A Tap::Support::Aggregator to collect the results of 
-    # methods that have no <tt>on_complete</tt> block
+    # methods that have no on_complete block
     attr_reader :aggregator
     
-    config :max_threads, 10, &c.integer           # For multithread execution
+    # A Tap::Support::Dependencies to track dependencies.
+    attr_reader :dependencies
+    
     config :debug, false, &c.flag                 # Flag debugging
     config :force, false, &c.flag                 # Force execution at checkpoints
     config :quiet, false, &c.flag                 # Suppress logging
-
+    config :verbose, false, &c.flag               # Enables extra logging (overrides quiet)
+    
     # The constants defining the possible App states.  
     module State
       READY = 0
@@ -222,18 +214,16 @@ module Tap
       super()
       
       @state = State::READY
-      @threads = [].extend(MonitorMixin)
-      @thread_queue = nil
-      @run_thread = nil
-      
       @queue = Support::ExecutableQueue.new
       @aggregator = Support::Aggregator.new
+      @dependencies = Support::Dependencies.new
       
       initialize_config(config)
       self.logger = logger
     end
     
-    DEFAULT_LOGGER = Logger.new(STDOUT)
+    # The default App logger writes to $stdout at level INFO.
+    DEFAULT_LOGGER = Logger.new($stdout)
     DEFAULT_LOGGER.level = Logger::INFO
     DEFAULT_LOGGER.formatter = lambda do |severity, time, progname, msg|
       "  %s[%s] %18s %s\n" % [severity[0,1], time.strftime('%H:%M:%S') , progname || '--' , msg]
@@ -257,7 +247,7 @@ module Tap
     # Logs the action and message at the input level (default INFO).  
     # Logging is suppressed if quiet is true.
     def log(action, msg="", level=Logger::INFO)
-      logger.add(level, msg, action.to_s) unless quiet
+      logger.add(level, msg, action.to_s) if !quiet || verbose
     end
     
     # Returns the configuration filepath for the specified task name,
@@ -267,170 +257,46 @@ module Tap
       name == nil ? nil : filepath('config', "#{name}.yml")
     end
     
-    #
-    # Execution methods
-    #
-    
-    # Executes the input Executable with the inputs.  Stores the result in 
-    # aggregator unless an on_complete block is set.  Returns the audited 
-    # result.
-    def execute(m, inputs)
-      _result = m._execute(*inputs)
-      aggregator.store(_result) unless m.on_complete_block
-      _result
-    end
-
-    # Sets state = State::READY unless the app has a run_thread 
-    # (ie the app is running).  Returns self.
+    # Sets state = State::READY unless the app is running.  Returns self.
     def ready
-      synchronize do 
-        self.state = State::READY if self.run_thread == nil
-        self
-      end
+      @state = State::READY unless state == State::RUN
+      self
     end
 
-    # Sequentially executes the methods (ie Executable objects) in queue; run 
-    # continues until the queue is empty and then returns self.  An app can 
-    # only run on one thread at a time.  If run is called when already running, 
-    # run returns immediately.
-    #
-    # === The Run Cycle
-    # Run can execute methods in sequential or multithreaded mode.  In sequential 
-    # mode, run executes enqued methods in order and on the current thread.  Run 
-    # continues until it reaches a method marked with multithread = true, at which 
-    # point run switches into multithreading mode.
-    #
-    # When multithreading, run shifts methods off of the queue and executes each 
-    # on their own thread (launching up to max_threads threads at one time). 
-    # Multithread execution continues until run reaches a non-multithread method,
-    # at which point run blocks, waits for the threads to complete, and switches 
-    # back into sequential mode.
-    #
-    # Run never executes multithreaded and non-multithreaded methods at the same 
-    # time.
-    #
-    # ==== Checks
-    # Run checks the state of self before executing a method.  If the state is 
-    # changed to State::STOP, then no more methods will be executed; currently 
-    # running methods will continute to completion.  If the state is changed to 
-    # State::TERMINATE then no more methods will be executed and currently running 
-    # methods will be discontinued as described below.
-    #
-    # ==== Error Handling and Termination
-    # When unhandled errors arise during run, run enters a termination routine.  
-    # During termination a TerminationError is raised in each executing method so 
-    # that the method exits or begins executing its internal error handling code 
-    # (perhaps performing rollbacks).
+    # Sequentially calls execute with the [executable, inputs] pairs in
+    # queue; run continues until the queue is empty and then returns self.
     #
-    # The TerminationError can ONLY be raised by the method itself, usually via a
-    # call to Tap::Support::Framework#check_terminate.  <tt>check_terminate</tt>
-    # is available to all Framework objects (ex Task and Workflow), but not to 
-    # Executable methods generated by _method.  These methods need to check the 
-    # state of app themselves; otherwise they will continue on to completion even 
-    # when app is in State::TERMINATE.
+    # ==== Run State
     #
-    #   # this task will loop until app.terminate
-    #   Task.new {|task|  while(true) task.check_terminate end }
+    # Run checks the state of self before executing a method. If state
+    # changes from State::RUN, the following behaviors result:
+    # 
+    # State::STOP:: No more executables will be executed; the current
+    #               executable will continute to completion.
+    # State::TERMINATE:: No more executables will be executed and the
+    #                    currently running executable will be
+    #                    discontinued as described in terminate.
     #
-    #   # this task will NEVER terminate
-    #   Task.new {|task|  while(true) end; task.check_terminate }
-    #
-    # Additional errors that arise during termination are collected and packaged 
-    # with the orignal error into a RunError.  By default all errors are logged
-    # and run exits.  If debug? is true, then the RunError will be raised for 
-    # further handling.
-    #
-    # Note: the method that caused the original unhandled error is no longer 
-    # executing when termination begins and thus will not recieve a 
-    # TerminationError.  
+    # Calls to run when the state is not State::READY do nothing and
+    # return immediately.
     def run
-      synchronize do
-        return self unless self.ready.state == State::READY
+      return self unless state == State::READY
+      @state = State::RUN
 
-        self.run_thread = Thread.current
-        self.state = State::RUN
-      end
-      
-      # generate threading variables
-      self.thread_queue = max_threads > 0 ? Queue.new : nil
-      
       # TODO: log starting run
-      begin 
-        execution_loop do
-          break if block_given? && yield(self) 
-          
-          # if no tasks were in the queue 
-          # then clear the threads and 
-          # check for tasks again
-          if queue.empty?
-            clear_threads 
-            # break -- no executable task was found
-            break if queue.empty?
-          end
-          
-          m, inputs = queue.deq
-          
-          if thread_queue && m.multithread
-            # TODO: log enqueuing task to thread
-            
-            # generate threads as needed and allowed
-            # to execute the threads in the thread queue
-            start_thread if threads.size < max_threads 
-
-            # NOTE: the producer-consumer relationship of execution
-            # threads and the thread_queue means that tasks will sit
-            # waiting until an execution thread opens up.  in the most
-            # extreme case all executing tasks and all tasks in the 
-            # task_queue could be the same task, each with different
-            # inputs.  this deviates from the idea of batch processing,
-            # but should be rare and not at all fatal given execute
-            # synchronization.  
-            thread_queue.enq [m, inputs]
-          
-          else
-            # TODO: log execute task
-            
-            # wait for threads to complete
-            # before executing the main thread
-            clear_threads
-            execute(m, inputs)
-          end
-        end
-        
-        # if the run loop exited due to a STOP state,
-        # tasks may still be in the thread queue and/or
-        # running.  be sure these are cleared
-        clear_thread_queue 
-        clear_threads 
-
-      rescue
-        # when an error is generated, be sure to terminate
-        # all threads so they can clean up after themselves.
-        # clear the thread queue first so no more tasks are
-        # executed. collect any errors that arise during
-        # termination. 
-        clear_thread_queue
-        errors =  [$!] + clear_threads(false)
-        errors.delete_if {|error| error.kind_of?(TerminateError) }
-        
-        # handle the errors accordingly
-        case
-        when debug?
-          raise Tap::Support::RunError.new(errors)
-        else
-          errors.each_with_index do |err, index|
-            log("RunError [#{index}] #{err.class}", err.message)
-          end
+      begin
+        until queue.empty? || state != State::RUN
+          executable, inputs = queue.deq
+          executable._execute(*inputs)
         end
+      rescue(TerminateError)
+        # gracefully fail for termination errors
+      rescue(Exception)
+        # handle other errors accordingly
+        raise if debug?
+        log($!.class, $!.message)
       ensure
-      
-        # reset run variables
-        self.thread_queue = nil
-        
-        synchronize do
-          self.run_thread = nil
-          self.state = State::READY
-        end
+        @state = State::READY
       end
       
       # TODO: log run complete
@@ -438,67 +304,46 @@ module Tap
     end
     
     # Signals a running application to stop executing tasks in the 
-    # queue by setting state = State::STOP.  Currently executing 
-    # tasks will continue their execution uninterrupted.
+    # queue by setting state = State::STOP.  The task currently 
+    # executing will continue uninterrupted to completion.
     #
     # Does nothing unless state is State::RUN.
     def stop
-      synchronize do
-        self.state = State::STOP if self.state == State::RUN
-        self
-      end
+      @state = State::STOP if state == State::RUN
+      self
     end
 
-    # Signals a running application to terminate executing tasks 
-    # by setting state = State::TERMINATE.  When running tasks
-    # reach a termination check, the task raises a TerminationError,
-    # thus allowing executing tasks to invoke their specific 
-    # error handling code, perhaps performing rollbacks.
-    #
-    # Termination checks can be manually specified in a task
-    # using the check_terminate method (see Tap::Task#check_terminate).
-    # Termination checks automatically occur before each task execution.
+    # Signals a running application to terminate execution by setting 
+    # state = State::TERMINATE.  In this state, an executing task 
+    # will then raise a TerminateError upon check_terminate, thus 
+    # allowing the invocation of task-specific termination, perhaps 
+    # performing rollbacks. (see Tap::Support::Executable#check_terminate).
     #
     # Does nothing if state == State::READY.
     def terminate
-      synchronize do
-        self.state = State::TERMINATE unless self.state == State::READY
-        self
-      end
+      @state = State::TERMINATE unless state == State::READY
+      self
     end
     
     # Returns an information string for the App.  
     #
-    #   App.instance.info   # => 'state: 0 (READY) queue: 0 thread_queue: 0 threads: 0 results: 0'
+    #   App.instance.info   # => 'state: 0 (READY) queue: 0 results: 0'
     #
-    # Provided information:
-    #
-    # state:: the integer and string values of self.state
-    # queue:: the number of methods currently in the queue
-    # thread_queue:: number of objects in the thread queue, waiting
-    #                to be run on an execution thread (methods, and 
-    #                perhaps nils to signal threads to clear)
-    # threads:: the number of execution threads
-    # results:: the total number of results in aggregator
     def info
-      synchronize do
-        "state: #{state} (#{State.state_str(state)}) queue: #{queue.size} thread_queue: #{thread_queue ? thread_queue.size : 0} threads: #{threads.size} results: #{aggregator.size}"
-      end
+      "state: #{state} (#{State.state_str(state)}) queue: #{queue.size} results: #{aggregator.size}"
     end
     
-    # Enques the task with the inputs.  If the task is batched, then each 
-    # task in task.batch will be enqued with the inputs.  Returns task.
-    #
-    # An Executable may provided instead of a task.
+    # Enques the task (or Executable) with the inputs.  If the task is batched, 
+    # then each task in task.batch will be enqued with the inputs.  Returns task.
     def enq(task, *inputs)
       case task
-      when Tap::Task, Tap::Workflow
-        raise "not assigned to enqueing app: #{task}" unless task.app == self
+      when Tap::Task
+        raise ArgumentError, "not assigned to enqueing app: #{task}" unless task.app == self
         task.enq(*inputs)
       when Support::Executable
         queue.enq(task, inputs)
       else
-        raise "Not a Task or Executable: #{task}"
+        raise ArgumentError, "not a Task or Executable: #{task}"
       end
       task
     end
@@ -509,60 +354,7 @@ module Tap
       m = object._method(method_name)
       enq(m, *inputs)
     end
-    
-    # Sets a sequence workflow pattern for the tasks such that the
-    # completion of a task enqueues the next task with it's results.
-    # Batched tasks will have the pattern set for each task in the 
-    # batch.  The current audited results are yielded to the block, 
-    # if given, before the next task is enqued.
-    #
-    # Executables may provided as well as tasks.
-    def sequence(*tasks) # :yields: _result
-      current_task = tasks.shift
-      tasks.each do |next_task|
-        # simply pass results from one task to the next.  
-        current_task.on_complete do |_result| 
-          yield(_result) if block_given?
-          enq(next_task, _result)
-        end
-        current_task = next_task
-      end
-    end
-
-    # Sets a fork workflow pattern for the tasks such that each of the
-    # targets will be enqueued with the results of the source when the
-    # source completes. Batched tasks will have the pattern set for each 
-    # task in the batch.  The source audited results are yielded to the 
-    # block, if given, before the targets are enqued.
-    #
-    # Executables may provided as well as tasks.
-    def fork(source, *targets) # :yields: _result
-      source.on_complete do |_result|
-        targets.each do |target| 
-          yield(_result) if block_given?
-          enq(target, _result)
-        end
-      end
-    end
-
-    # Sets a merge workflow pattern for the tasks such that the results
-    # of each source will be enqueued to the target when the source 
-    # completes. Batched tasks will have the pattern set for each 
-    # task in the batch.  The source audited results are yielded to  
-    # the block, if given, before the target is enqued.
-    #
-    # Executables may provided as well as tasks.
-    def merge(target, *sources) # :yields: _result
-      sources.each do |source|
-        # merging can use the existing audit trails... each distinct 
-        # input is getting sent to one place (the target)
-        source.on_complete do |_result| 
-          yield(_result) if block_given?
-          enq(target, _result)
-        end
-      end
-    end
-    
+        
     # Returns all aggregated, audited results for the specified tasks.  
     # Results are joined into a single array.  Arrays of tasks are 
     # allowed as inputs. See results.
@@ -573,147 +365,28 @@ module Tap
     # Returns all aggregated results for the specified tasks.  Results are
     # joined into a single array.  Arrays of tasks are allowed as inputs.    
     #
-    #   t1 = Task.new  {|task, input| input += 1 }
-    #   t2 = Task.new  {|task, input| input += 10 }
-    #   t3 = t2.initialize_batch_obj
+    #   t0 = Task.intern  {|task, input| "#{input}.0" }
+    #   t1 = Task.intern  {|task, input| "#{input}.1" }
+    #   t2 = Task.intern  {|task, input| "#{input}.2" }
+    #   t1.batch_with(t2)
     #
-    #   t1.enq(0)
-    #   t2.enq(1)
+    #   t0.enq(0)
+    #   t1.enq(1)
     #
     #   app.run
-    #   app.results(t1, t2.batch)     # => [1, 11, 11]
-    #   app.results(t2, t1)           # => [11, 1]
+    #   app.results(t0, t1.batch)     # => ["0.0", "1.1", "1.2"]
+    #   app.results(t1, t0)           # => ["1.1", "0.0"]
     #
     def results(*tasks)
       _results(tasks).collect {|_result| _result._current}
     end
     
-    protected
-    
-    # A hook for handling unknown configurations in subclasses, called from
-    # configure.  If handle_configuration evaluates to false, then configure
-    # raises an error.
-    def handle_configuation(key, value)
-      false
+    def inspect
+      "#<#{self.class.to_s}:#{object_id} root: #{root} >"
     end
     
-    # Sets the state of the application
-    attr_writer :state
-    
-    # The thread on which run is executing tasks.
-    attr_accessor :run_thread
-    
-    # An array containing the execution threads in use by run.  
-    attr_accessor :threads
-    
-    # A Queue containing multithread tasks waiting to be run 
-    # on the execution threads.  Nil if max_threads == 0
-    attr_accessor :thread_queue
-    
-    private
-
-    def execution_loop
-      while true
-        case state
-        when State::STOP
-          break
-        when State::TERMINATE
-          # if an execution thread handles the termination error, 
-          # then the thread may end up here -- terminated but still 
-          # running.  Raise another termination error to enter the 
-          # termination (rescue) code.
-          raise TerminateError.new
-        end
-   
-        yield
-      end
-    end
-
-    def clear_thread_queue
-      return unless thread_queue
-      
-      # clear the queue and enque the thread complete
-      # signals, so that the thread will exit normally
-      dequeued = []
-      while !thread_queue.empty?
-        dequeued << thread_queue.deq
-      end
-
-      # add dequeued tasks back, in order, to the task 
-      # queue so no tasks get lost due to the stop
-      #
-      # BUG: this will result in an already-newly-queued 
-      # task being promoted along with it's inputs
-      dequeued.reverse_each do |task, inputs|
-        # TODO: log about not executing
-        queue.unshift(task, inputs) unless task.nil?
-      end
-    end
-    
-    def clear_threads(raise_errors=true)
-      threads.synchronize do
-        errors = []
-        return errors if threads.empty?
-      
-        # clears threads gracefully by enqueuing nils, to break
-        # the threads out of their loops, then waiting for the
-        # threads to work through the queue to the nils
-        #
-        threads.size.times { thread_queue.enq nil }
-        while true
-          # TODO -- add a time out?
-          
-          threads.dup.each do |thread|
-            next if thread.alive?
-            threads.delete(thread)
-            error = thread["error"]
-            
-            next if error.nil?
-            raise error if raise_errors
-            
-            errors << error
-          end
-
-          break if threads.empty?
-          Thread.pass
-        end
-      
-        errors
-      end
-    end
-    
-    def start_thread
-      threads.synchronize do
-        # start a new thread and add it to threads.
-        # threads simply loop and wait for a task to 
-        # be queued.  the thread will block until a 
-        # task is available (due to thread_queue.deq)
-        #
-        # TODO -- track thread index like?
-        #  thread["index"] = threads.length
-        threads << Thread.new do 
-          # TODO - log thread start
-        
-          begin
-            execution_loop do
-              m, inputs = thread_queue.deq
-              break if m.nil?
-            
-              # TODO: log execute task on thread #
-              execute(m, inputs)
-            end
-          rescue
-            # an unhandled error should immediately
-            # terminate all threads
-            terminate
-            Thread.current["error"] = $!
-          end
-        end
-      end
-    end
-
-    # TerminateErrors are raised to kill executing tasks when terminate 
-    # is called on an running App.  They are handled by the run rescue code.
+    # TerminateErrors are raised to kill executing tasks when terminate is 
+    # called on an running App.  They are handled by the run rescue code.
     class TerminateError < RuntimeError 
     end
   end