tap 0.9.1 → 0.10.0

data/cmd/destroy.rb

@@ -0,0 +1,16 @@
+require 'tap/generator/base'
+require 'tap/generator/destroy'
+
+env = Tap::Env.instance
+
+if ARGV.empty? || ARGV == ['-T']
+  puts env.summarize(:generators) {|const| const.document[const.name]['generator'] }
+  exit
+end
+
+name = ARGV.shift
+const = env.search(:generators, name) or raise "unknown generator: #{name}"
+
+generator_class = const.constantize
+generator, argv = generator_class.instantiate(ARGV)
+generator.extend(Tap::Generator::Destroy).process(*argv)

data/cmd/generate.rb

@@ -0,0 +1,16 @@
+require 'tap/generator/base'
+require 'tap/generator/generate'
+
+env = Tap::Env.instance
+
+if ARGV.empty? || ARGV == ['-T']
+  puts env.summarize(:generators) {|const| const.document[const.name]['generator'] }
+  exit
+end
+
+name = ARGV.shift
+const = env.search(:generators, name) or raise "unknown generator: #{name}"
+
+generator_class = const.constantize
+generator, argv = generator_class.instantiate(ARGV)
+generator.extend(Tap::Generator::Generate).process(*argv)

data/cmd/run.rb

@@ -0,0 +1,126 @@
+# tap run {options} -- {task options} task INPUTS...
+#
+# examples:
+#   tap run --help                     Prints this help
+#   tap run -- task --help             Prints help for task
+#
+
+env = Tap::Env.instance
+app = Tap::App.instance
+cmdline = Tap::Support::CommandLine
+
+#
+# handle options
+#
+
+dump = false
+OptionParser.new do |opts|
+  
+  opts.separator ""
+  opts.separator "configurations:"
+  
+  Tap::App.configurations.each do |receiver, key, config|
+    next if receiver == Tap::Root
+    
+    opts.on(*cmdline.configv(config)) do |value|
+      app.send(configuration.writer, value)
+    end
+  end
+ 
+  opts.separator ""
+  opts.separator "options:"
+
+  opts.on("-h", "--help", "Show this message") do
+    opts.banner = cmdline.usage(__FILE__)
+    Tap::App.lazydoc.resolve
+    puts opts
+    exit
+  end
+  
+  opts.on('-T', '--manifest', 'Print a list of available tasks') do |v|
+    puts env.summarize(:tasks) {|const| const.document[const.name]['manifest'] }
+    exit
+  end
+  
+end.parse!(ARGV)
+
+#
+# handle options for each specified task
+#
+
+rounds = cmdline.split(ARGV).collect do |argvs|
+  argvs.each do |argv|
+    ARGV.clear  
+    ARGV.concat(argv)
+   
+    unless td = cmdline.shift(ARGV)
+      # warn nil?
+      next
+    end
+
+    # attempt lookup the task class
+    const = env.search(:tasks, td) or raise "unknown task: #{td}"
+    task_class = const.constantize or raise "unknown task: #{td}"
+    
+    # now let the class handle the argv
+    task, argv = task_class.instantiate(ARGV, app)
+    task.enq *argv.collect! {|str| cmdline.parse_yaml(str) }
+  end
+
+  app.queue.clear
+end
+ARGV.clear
+
+rounds.delete_if {|round| round.empty? }
+if rounds.empty?
+  puts "no task specified"
+  exit
+end
+
+#
+# set signals 
+#
+
+# info signal -- Note: some systems do 
+# not support the INFO signal 
+# (windows, fedora, at least)
+signals = Signal.list.keys
+if signals.include?("INFO")
+  Signal.trap("INFO") do
+    puts app.info
+  end
+end
+
+# interuption signal
+if signals.include?("INT")
+  Signal.trap("INT") do
+    puts " interrupted!"
+    # prompt for decision
+    while true
+      print "stop, terminate, exit, or resume? (s/t/e/r):"
+      case gets.strip
+      when /s(top)?/i 
+        app.stop
+        break
+      when /t(erminate)?/i 
+        app.terminate
+        break
+      when /e(xit)?/i 
+        exit
+      when /r(esume)?/i 
+        break
+      else
+        puts "unexpected response..."
+      end
+    end
+  end
+end
+
+#
+# enque tasks and run!
+#
+
+rounds.each_with_index do |queue, i|
+  app.queue.concat(queue)
+  app.run
+end

data/doc/Class Reference

@@ -0,0 +1,362 @@
+= Class Reference
+
+Working up from the ground is useful to get a sense for how Tap does what it does.  This reference goes through the modules and classes that build up a task application: Tasks, Apps, and Envs.
+
+== Tasks (Tap::Task, Tap::Workflow)
+
+==== Methods
+
+http://tap.rubyforge.org/images/Method.png
+
+Tasks begin with methods, simply a block of code.
+
+==== Tap::Support::Executable
+
+http://tap.rubyforge.org/images/Executable.png
+
+Executable extends objects allowing them to be enqued and run by an App.  Executable objects specify a method that gets called upon execution; in essence Executable wraps this method and adds workflow behaviors like auditing and an <tt>on_complete</tt> block.
+
+Tasks are constructed such that <tt>execute</tt> is their executable method.  Task#execute simply provides hooks before and after forwarding control to the <tt>process</tt> method.  Hence <tt>process</tt> is the standard method overridden in subclasses of Task.
+
+==== Tap::Support::Batchable
+
+http://tap.rubyforge.org/images/Batchable.png
+
+Tasks can be assembled into batches that all enque at the same time and share the same <tt>on_complete</tt> block.  This behavior is very powerful, allowing workflow logic to be defined once but executed under multiple conditions.  Task includes the Batchable module to facilitate batching.
+
+==== Tap::Support::Configurable
+
+http://tap.rubyforge.org/images/Configurable.png
+
+Configurable allows declaration of class configurations.  Configurable classes have a <tt>configurations</tt> method accessing a {ClassConfiguration}[link:classes/Tap/Support/ClassConfiguration.html] which holds all declared configs, their default values, and metadata for transforming configurations into command line options (for example).  
+
+Instances of a Configurable class have a <tt>config</tt> method accessing a {InstanceConfiguration}[link:classes/Tap/Support/InstanceConfiguration.html] object.  The instance configuration acts like a forwarding hash; read and write operations for declared configs get forwarded to class methods while undeclared configs are stored directly.  The writer for a config may be defined through a block provided during config declaration.  For instance:
+
+  class ConfigClass
+    include Tap::Support::Configurable
+
+    config :key, 'value' do |input|
+      input.upcase
+    end
+  end
+
+Is basically the same as:
+
+  class RegularClass
+    attr_reader :key
+
+    def key=(input)
+      @key = input.upcase
+    end
+
+    def initialize
+      self.key = 'value'
+    end
+  end
+
+As you can see here:
+
+  c = ConfigClass.new
+  c.key                       # => 'VALUE'
+
+  c.config[:key] = 'new value'
+  c.key                       # => 'NEW VALUE'
+
+  c.key = 'another value'
+  c.config[:key]              # => 'ANOTHER VALUE'
+
+This setup is both fast and convenient.  
+
+==== Tap::Support::Validation
+
+When configurations are set from the command line, the writer method will inevitably receive a string, whereas configurations set within code can receive any type of object.  The {Validation}[link:classes/Tap/Support/Validation.html] module provides standard blocks for validating and transforming inputs, accessible through the <tt>c</tt> method (ex: <tt>c.integer</tt> or <tt>c.regexp</tt>).  These blocks (generally) load string inputs as YAML and validate that the result is the correct class; non-string inputs are simply validated.
+
+  class ValidatingClass
+    include Tap::Support::Configurable
+
+    config :int, 1, &c.integer                 # assures the input is an integer
+    config :int_or_nil, 1, &c.integer_or_nil   # integer or nil only
+    config :array, [], &c.array                # you get the idea
+  end
+
+  vc = ValidatingClass.new
+
+  vc.array = [:a, :b, :c]
+  vc.array                                     # => [:a, :b, :c]
+
+  vc.array = "[1, 2, 3]"
+  vc.array                                     # => [1, 2, 3]
+
+  vc.array = "string"                          # !> ValidationError
+
+Validation blocks sometimes imply metadata.  For instance <tt>c.flag</tt> makes a config into a flag on the command line.
+
+==== Tap::Support::Lazydoc
+
+http://tap.rubyforge.org/images/Lazydoc.png
+
+Ah lazydoc.  Lazydoc fits into the space between live code and code documentation.  Lazydoc can scan a file (code or not) and pull out documentation into the object space where it can be utilized.  Lazydoc uses a key-value syntax that is both invalid Ruby and easily hidden in RDoc.  Looks like this:
+
+  # ::key value
+
+Try the former line without the comment and a syntax error results.  In RDoc, the syntax doesn't interfere with any of the standard documentation conventions and can be hidden with the use of a <tt>:startdoc:</tt> attribute (an extra space is added to <tt>:startdoc:</tt> so the line isn't actually hidden in this document):
+
+  # :start doc::key value
+
+Lazydoc parses a constant name, the key, the value, and any comment following the value until a non-comment line or an end key.  For example:
+
+  [lazydoc_file.rb]
+  # Name::Space::key value
+  # 
+  # This documentation
+  # gets parsed.
+  #
+
+  # Name::Space::another another value
+  # This gets parsed.
+  # Name::Space::another-
+  #
+  # This does not.
+
+  require 'tap'
+
+  lazydoc = Tap::Support::Lazydoc[__FILE__]
+  lazydoc.resolve
+
+  lazydoc['Name::Space']['key'].to_s           # => "This documentation gets parsed."
+  lazydoc['Name::Space']['another'].subject    # => "another value"
+
+Furthermore, Lazydoc allows living code to register lines that should get documented.  These lines are parsed to echo what happens in RDoc.
+
+  [another_lazydoc_file.rb]
+  # documentation
+  # for the method
+  def method
+  end
+
+  require 'tap'
+
+  lazydoc = Tap::Support::Lazydoc[__FILE__]
+  code_comment = lazydoc.register(2)
+  lazydoc.resolve
+  
+  code_comment.subject         # => "def method"
+  code_comment.to_s            # => "documentation for the method"
+
+Tap uses Lazydoc to indicate when a file contains a Task (<tt>::manifest</tt>) or a generator (<tt>::generator</tt>), and for config documentation.  Tap::Env uses this information to facilitate lookup and instantiation of task classes.  
+
+One note: when no constant name is specified for a Lazydoc key, it gets treated as a default for the whole file. 
+
+  [lib/sample/task.rb]
+  # ::manifest sample task description
+  #
+  # This manifest is expected to apply to the Sample::Task class.
+  # If more than one task is defined in this file, or if Sample::Task
+  # is not defined by loading this file, Tap will run into trouble.
+
+However, the best practice is to include the namespace explicitly.
+
+=== Tap::Task
+
+http://tap.rubyforge.org/images/Task.png
+
+Running a task through the tap executable instantiates a task class, configures it, enques it, and runs a Tap::App to get the inputs to the <tt>process</tt> method.  Tasks do not have to be used this way; they are perfectly capable as objects in free-standing scripts.
+
+Task instances can take a block that acts as a stand-in for <tt>process</tt>:
+
+  t = Tap::Task.new {|task| 1 + 2 }
+  t.process                # => 3
+
+  t = Tap::Task.new {|task, x, y| x + y }
+  t.process(1, 2)          # => 3
+ 
+Tasks can be configured:
+
+  t1 = Tap::Task.new(:key => 'one') {|task, input| "#{input}:#{task.config[:key]}"}
+  t1.process('str')        # => "str:one"
+  
+And batched:
+
+  t2 = t1.initialize_batch_obj(:key => 'two')
+  t3 = t1.initialize_batch_obj(:key => 'three')
+
+  t1.batch                 # => [t1, t2, t3]
+
+Batched tasks enque together, and therefore run sequentially with the same inputs:
+
+  app = Tap::App.instance
+  app.enq(t1, 'str')
+  app.queue.to_a           # => [[t1, ['str']], [t2, ['str']], [t3, ['str']]]
+  app.run
+
+  app.results(t1)          # => ["str:one"]
+  app.results(t2)          # => ["str:two"]
+  app.results(t3)          # => ["str:three"]
+
+Also, as a consequence of Task being Executable, the results have audit trails.  In the audit trail, the tasks are identified by name (by default the name the underscored class name):
+
+  t1.name = 'task one'
+  t2.name = 'task two'
+  t3.name = 'task three'
+  
+  app._results(t1,t2,t3).collect do |_result|
+    _result.to_s
+  end.join("---\n")
+  # => 
+  # o-[] 1
+  # o-[task one] "str:one"
+  # ---
+  # o-[] 1
+  # o-[task two] "str:two"
+  # ---
+  # o-[] 1
+  # o-[task three] "str:three"
+  
+Task instances can be joined into workflows.  The workflow model used by Tap is very simple; when a task completes it calls its <tt>on_complete</tt> block to enque the next task (or tasks) in the workflow.  Arbitrary workflow logic is allowed since there are no restrictions on what the <tt>on_complete</tt> block does.  If a task has no <tt>on_complete</tt> block, App collects the unhandled results (as shown above) so they can be handled somewhere else.  See below for more details.
+
+=== Tap::Workflow
+
+http://tap.rubyforge.org/images/Workflow.png
+
+Workflows are not Tasks but are constructed with the same modules as Task and work very similarly Tasks.  Workflows have a <tt>workflow</tt> method which defines the entry and exit points for the workflow; there can be 1+ entry points and 0+ exit points.  The enque method for a workflow enques all it's entry points, and when the <tt>on_complete</tt> block is set for a workflow, it is set for all exit points.
+
+Like Tasks, Workflows can be configured, enqued, used in workflows, and subclassed.
+
+== Apps
+
+==== Tap::Root
+
+http://tap.rubyforge.org/images/Root.png
+
+A Root represents the base of a directory structure. Roots allow you to alias directories and ease working with filepaths, basically allowing you to develop code for a conceptual directory structure that can be defined later.
+
+  root = Tap::Root.new '/path/to/root'
+  root.root                                      # => '/path/to/root'
+  root['config']                                 # => '/path/to/root/config'
+  root.filepath('config', 'sample.yml')          # => '/path/to/root/config/sampl.yml'
+
+While simple, this ability to reference files using aliases is useful, powerful, and forms the basis of the Tap execution environment.
+
+==== Tap::Support::ExecutableQueue
+
+http://tap.rubyforge.org/images/ExecutableQueue.png
+
+Apps coordinate the execution of tasks through a queue.  The queue is just a stack of Executable objects, basically methods, and the inputs to those methods; during a run the enqued methods are sequentially executed with the inputs.  
+
+Normally the methods execute one after the other on the main thread, however the methods can be set to multithread.  In this case all sequential, multithreadable methods are executed on their own thread (up to a configurable maximum number of threads).  A non-multithreadable method blocks until all these threads finish, and then the sequential, single thread execution resumes.  
+
+For instance, assuming non-multithreadable executables [a,b,c] and multithread executables [m1, m2, m3], a queue of [a, m1, m2, m3, b, c] executes like:
+
+  a... done
+  (m1, m2, m3)... done
+  b... done
+  c... done
+
+==== Tap::Support::Audit
+
+Tap tracks inputs as they are modified by various tasks, again through Executable.  At the end of a run, any individual result can be tracked back to it's original value with references to the source of each change in the value (ie the task or Executable).  This auditing can be very useful when workflows diverge, as they often do.
+
+Auditing is largely invisible except in <tt>on_complete</tt> blocks.  <tt>on_complete</tt> blocks receive the audited results so that this information can be used, as needed, to make decisions.  
+
+    t = Task.new
+    t.on_complete do |_result|      # _result is an Audit instance
+      _result._current              # the current value
+      _result._original             # the original value
+    end
+
+To help indicate when a result is actually a result and when it is an audit, Tap uses a convention whereby a leading underscore signals auditing is involved.
+
+==== Tap::Support::Aggregator
+
+When a task completes, it executes it's <tt>on_complete</tt> block to handle the results, perhaps passing them on to other tasks.  Aggregators collect results when no <tt>on_complete</tt> block is specified.  Results are collected per-task into an array; a single task executed many times will have it's results aggregated into this single array.
+
+=== Tap::App
+
+http://tap.rubyforge.org/images/App.png
+
+Instances of Tap::App coordinate the execution of tasks.  Apps are basically a subclass of Root with an ExecutableQueue and Aggregator.  Task initialization requires an App, which is by default Tap::App.instance.  Tasks use their app for logging, checks, and to enque themselves.  Normally a script will only need and use a single instance (often Tap::App.instance), but there is no reason why multiple instances could not be used.  
+
+  log = StringIO.new
+  app = Tap::App.instance
+  app.logger = Logger.new(log)
+  
+  t = Tap::Task.new
+  t.log 'action', 'to app'
+  log.string                 # =>  "  I[15:21:23]             action  to app\n"
+  
+  t.enq(1)
+  t.enq(2,3)
+  app.queue.to_a             # => [[t, [1]], [t, [2,3]]
+  
+Apps also coordinate the creation of standard workflow patterns like sequence, fork, and merge.  These methods set <tt>on_complete</tt> blocks for the input tasks.
+
+  t1 = Tap.task('t1') {|t| 'hellO'}
+  t2 = Tap.task('t2') {|t, input| input + ' woRld' }
+  t3 = Tap.task('t3') {|t, input| input.downcase }
+  t4 = Tap.task('t4') {|t, input| input.upcase }
+  t5 = Tap.task('t5') {|t, input| input + "!" }
+  
+  # sequence t1, t2
+  app.sequence(t1, t2)
+  
+  # fork t2 results to t3 and t4
+  app.fork(t2, t3, t4)
+  
+  # unsynchronized merge of t3 and t4 into t5
+  app.merge(t5, t3, t4)
+  
+  app.enq(t1)
+  app.run
+  
+  app.results(t5)       # => ["hello world!", "HELLO WORLD!"]
+
+As shown above, aggregated results may be accessed by task through the <tt>results</tt> method. To access the audited results, use <tt>_results</tt>:
+
+  app._results(t5).collect do |_result|
+    _result.to_s
+  end.join("---\n")
+  # =>
+  # o-[t1] "hellO"
+  # o-[t2] "hellO woRld"
+  # o-[t3] "hello world"
+  # o-[t5] "hello world!"
+  # ----
+  # o-[t1] "hellO"
+  # o-[t2] "hellO woRld"
+  # o-[t4] "HELLO WORLD"
+  # o-[t5] "HELLO WORLD!"
+  
+== Envs 
+
+==== Tap::Env
+
+http://tap.rubyforge.org/images/Env.png
+
+Environments are still under construction.  Basically a wrapper for a Root, Envs define methods to generate manifests for a type of file-based resource (tasks, generators, etc).  Furthermore they provide methods to uniquely identify the resource by path or, more specifically, minimized base paths.  In this directory structure:
+
+  path
+  `- to
+      |- another
+      |   `- file.rb
+      |- file-0.1.0.rb
+      |- file-0.2.0.rb
+      `- file.rb
+
+The minimal paths that uniquely identify these files are (respectively):
+
+  'another/file'
+  'file-0.1.0'
+  'file-0.2.0'
+  'file.rb'
+
+Envs facilitate mapping the minimal path, which might be provided by the command line, to the actual path, and hence to the resource.  Envs can be nested so that manifests span multiple directories.  Indeed, this is how tap accesses tasks and generators within gems; the gem directories are initialized as Envs and nested within the Env for the working directory.
+
+http://tap.rubyforge.org/images/Nested-Env.png
+
+To prevent conflicts between similarly-named resources under two Envs, Env allows selection of Envs, also by minimized paths.  At present this is difficult to illustrate in a code snippit.
+
+--
+==== Run Env
+http://tap.rubyforge.org/images/Run-Env.png
+++
+