wukong 3.0.0.pre → 3.0.0.pre2

data/.gitignore

@@ -1,46 +1,59 @@
+## OS
+.DS_Store
+Icon
+nohup.out
+.bak
 
+*.pem
 
-
-
-
+## EDITORS
+\#*
+.\#*
+\#*\#
+*~
+*.swp
+REVISION
+TAGS*
+tmtags
+*_flymake.*
+*_flymake
+*.tmproj
+.project
+.settings
 
 ## COMPILED
-## EDITORS
-## OS
-## OTHER SCM
-## PROJECT::GENERAL
-## PROJECT::SPECIFIC
-*.log
+a.out
 *.o
 *.pyc
-*.rdb
 *.so
-*.swp
-*.tmproj
-*_flymake
-*_flymake.*
-*private*
-*~
-.DS_Store
-.\#*
-.bak
+
+## OTHER SCM
 .bzr
 .hg
-.project
-.settings
 .svn
-.yardoc
-/.rbenv-version
-Gemfile.lock
-Icon?
-REVISION
-TAGS*
-\#*
-a.out
+
+## PROJECT::GENERAL
+
+log/*
+tmp/*
+pkg/*
+
 coverage
+rdoc
 doc
-nohup.out
 pkg
-rdoc
-tmp/*
-tmtags
+.rake_test_cache
+.bundle
+.yardoc
+
+.vendor
+
+## PROJECT::SPECIFIC
+
+old/*
+docpages
+away
+
+.rbx
+Gemfile.lock
+Backup*of*.numbers

data/.gitmodules

@@ -1,3 +1,6 @@
 [submodule "notes"]
 	path = notes
 	url = git://github.com/infochimps-labs/wukong.wiki.git
+[submodule "data"]
+	path = data
+	url = http://github.com/infochimps-labs/wukong-example-data.git

data/.rspec

@@ -1,2 +1,2 @@
+--format=progress
 --color
---format documentation

data/.travis.yml

@@ -1,12 +1,19 @@
 language: ruby
 rvm:
   - 1.9.3
+  - 1.9.2
+  - jruby-19mode # JRuby in 1.9 mode
+  - rbx-19mode
 
 before_install: "git clone -b version_1 git://github.com/infochimps-labs/gorillib.git ~/builds/infochimps-labs/gorillib"
 
+bundler_args: --without docs support
+
 branches:
   only:
+    - master
     - wukong_ng
+    - vanilla_2
 
 notifications:
-  email: false
+  email: false

data/.yardopts

@@ -1,19 +1,6 @@
 --readme   README.md
 --markup   markdown
 -
-VERSION
 CHANGELOG.md
 LICENSE.md
 README.md
-notes/INSTALL.md
-notes/core_concepts.md
-notes/knife-cluster-commands.md
-notes/philosophy.md
-notes/silverware.md
-notes/style_guide.md
-notes/tips_and_troubleshooting.md
-notes/walkthrough-hadoop.md
-notes/homebase-layout.txt
-
-notes/*.md
-notes/*.txt

data/Guardfile

@@ -1,14 +1,12 @@
 # -*- ruby -*-
 
-format = 'progress' # 'doc' for more verbose, 'progress' for less
-tags   = %w[ ]      # builder_spec model_spec
+format = 'doc'          # 'doc' for more verbose, 'progress' for less
+tags   = %w[ only ]     # builder_spec model_spec
 
 guard 'rspec', :version => 2, :cli => "--format #{format} #{ tags.map{|tag| "--tag #{tag}"}.join(' ')  }" do
   watch(%r{^spec/.+_spec\.rb$})
-  watch(%r{^examples/(\w+)/(.+)\.rb$}) { |m| "spec/examples/#{m[1]}_spec.rb" }
-  watch(%r{^examples/(\w+)\.rb$})      { |m| "spec/examples/#{m[1]}_spec.rb" }
-  watch(%r{^lib/(.+)/(.+)\.rb$})       { |m| "spec/#{m[1]}/#{m[2]}_spec.rb"  }
-  watch(%r{^lib/(\w+)\.rb$})           { |m| "spec/#{m[1]}}_spec.rb"         }
+  watch(%r{^examples/(.+)\.rb$})       { |m| "spec/examples/#{m[1]}_spec.rb" }
+  watch(%r{^lib/(.+)\.rb$})            { |m| "spec/#{m[1]}}_spec.rb"         }
   watch('spec/spec_helper.rb')         { 'spec' }
   watch(/spec\/support\/(.+)\.rb/)     { 'spec' }
 end

data/{LICENSE.textile → LICENSE.md}

@@ -1,107 +1,95 @@
----
-layout: default
-title:  Apache License
----
+# License for Wukong
 
+The wukong code is __Copyright (c) 2011, 2012 Infochimps, Inc__
 
-h1(gemheader). {{ site.gemname }} %(small):: license%
-
-
-The wukong code is __Copyright (c) 2009 Philip (flip) Kromer__
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+This code is licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
 
  http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an **AS IS** BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 
-h1. Apache License
+__________________________________________________________________________
+
+# Apache License
+
 
                               Apache License
                         Version 2.0, January 2004
                      http://www.apache.org/licenses/
 
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-<notextile><div class="toggle"></notextile>
+_TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION_
 
-h2. 1. Definitions.
+## 1. Definitions.
 
 * **License** shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
 * **Licensor** shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
-* **Legal Entity** shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, **control** means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+* **Legal Entity** shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
 * **You** (or **Your**) shall mean an individual or Legal Entity exercising permissions granted by this License.
+
 * **Source** form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
 * **Object** form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
 * **Work** shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
-* **Derivative Works** shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
-* **Contribution** shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, **submitted** means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
-* **Contributor** shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
 
-<notextile></div><div class="toggle"></notextile>
+* **Derivative Works** shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
 
-h2. 2. Grant of Copyright License.
+* **Contribution** shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
 
-Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+* **Contributor** shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
 
+## 2. Grant of Copyright License.
 
-<notextile></div><div class="toggle"></notextile>
+Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
 
-h2. 3. Grant of Patent License.
+## 3. Grant of Patent License.
 
 Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
 
-<notextile></div><div class="toggle"></notextile>
-
-h2. 4. Redistribution.
+## 4. Redistribution.
 
 You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
 
-# You must give any other recipients of the Work or Derivative Works a copy of this License; and
-# You must cause any modified files to carry prominent notices stating that You changed the files; and
-# You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
-# If the Work includes a __NOTICE__ text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+  - (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
+  - (b) You must cause any modified files to carry prominent notices stating that You changed the files; and
+  - (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+  - (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
 
 You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
 
-<notextile></div><div class="toggle"></notextile>
-
-h2. 5. Submission of Contributions.
+## 5. Submission of Contributions.
 
 Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
 
-<notextile></div><div class="toggle"></notextile>
-
-h2. 6. Trademarks.
+## 6. Trademarks.
 
 This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
 
-<notextile></div><div class="toggle"></notextile>
+## 7. Disclaimer of Warranty.
 
-h2. 7. Disclaimer of Warranty.
+Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
 
-Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an **AS IS** BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
-
-<notextile></div><div class="toggle"></notextile>
-
-h2. 8. Limitation of Liability.
+## 8. Limitation of Liability.
 
 In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
 
-<notextile></div><div class="toggle"></notextile>
-
-h2. 9. Accepting Warranty or Additional Liability.
+## 9. Accepting Warranty or Additional Liability.
 
 While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
 
-END OF TERMS AND CONDITIONS
-
-<notextile></div><div class="toggle"></notextile>
+_END OF TERMS AND CONDITIONS_
 
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+## APPENDIX: How to apply the Apache License to your work.
 
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an **AS IS** BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets `[]` replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
 
-<notextile></div></notextile>
+> Copyright [yyyy] [name of copyright owner]
+>
+> Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+>
+>     http://www.apache.org/licenses/LICENSE-2.0
+>
+> Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

data/README-old.md

@@ -0,0 +1,422 @@
+# Wukong [![Build Status](https://secure.travis-ci.org/infochimps-labs/wukong.png)](http://travis-ci.org/infochimps-labs/wukong)
+
+Wukong is a toolkit for rapid, agile development of dataflows at any scale.
+
+(note: the syntax below is mostly false)
+
+
+<a name="design"></a>
+## Design Overview
+
+The fundamental principle of Wukong/Hanuman is *powerful black boxes, beautiful glue*. In general, they don't do the thing -- they coordinate the boxes that do the thing, to let you implement rapidly, nimbly and readably. Hanuman elegantly describes high-level data flows; Wukong is a pragmatic collection of dataflow primitives. They both emphasize scalability, readability and rapid development over performance or universality.
+
+Wukong/Hanuman are chiefly concerned with these specific types of graphs:
+
+* **dataflow**      -- chains of simple modules to handle continuous data processing -- coordinates Flume, Unix pipes, ZeroMQ, Esper, Storm.
+* **workflows**     -- episodic jobs sequences, joined by dependency links -- comparable to Rake, Azkaban or Oozie.
+* **map/reduce**    -- Hadoop's standard *disordered/partitioned stream > partition, sort & group > process groups* workflow. Comparable to MRJob and Dumbo.
+* **queue workers** -- pub/sub asynchronously triggered jobs -- comparable Resque, RabbitMQ/AMQP, Amazon Simple Worker, Heroku workers.
+
+In addition, wukong stages may be deployed into **http middlware**: lightweight distributed API handlers -- comparable to Rack, Goliath or Twisted.
+
+When you're describing a Wukong/Hanuman flow, you're writing pure expressive ruby, not some hokey interpreted language or clumsy XML format. Thanks to JRuby, it can speak directly to Java-based components like Hadoop, Flume, Storm or Spark.
+
+## What's where
+
+* Configliere -- Manage settings
+  - Layer - Project settings through a late-resolved stack of config objects.
+* Gorillib
+  - Type, RecordType
+  - TypeConversion
+  - Model
+  - PathHelpers
+* Wukong
+  - fs - Abstracts file hdfs s3n s3hdfs scp
+  - streamer - Black-box data transform
+  - job - Workflow definition
+  - flow - Dataflow definition
+  - widgets - Common data transforms
+  - RubyHadoop - Hadoop jobs using streamers
+  - RubyFlume - Flume decorators using streamers
+* Hanuman -- Elegant small graph assembly
+* Swineherd -- Common interface on ugly tools
+  - Turn readable hash into safe commandline (param conv, escaping)
+  - Execute command, capture stdin/stderr
+  - Summarize execution with a broham-able hash
+  - Common modules: Input/output, Java, gnu, configliere
+  - Template
+  - Hadoop, pig, flume
+  - ?? Cp, mv, rm, zip, tar, bz2, gz, ssh, scp
+  - ?? Remotely execute command
+
+<a name="design-rules"></a>
+### Story
+
+[Narrative Method Structure](http://avdi.org/talks/confident-code-rubymidwest-2011/confident-code.html)
+
+* Gather input
+* Perform work
+* Deliver results
+* Handle failure
+
+
+<a name="design-rules"></a>
+### Design Rules
+
+* **whiteboard rule**: the user-facing conceptual model should match the picture you would draw on the whiteboard in an engineering discussion. The fundamental goal is to abstract the necessary messiness surrounding the industrial-strength components it orchestrates while still providing their essential power.
+* **common cases are simple, complex cases are always possible**: The code should be as simple as the story it tells. For the things you do all the time, you only need to describe how this data flow is different from all other data flows. However, at no point in the project lifecycle should Wukong/Hanuman hit a brick wall or peat bog requiring its total replacement. A complex production system may, for example, require that you replace a critical path with custom Java code -- but that's a small set of substitutions in an otherwise stable, scalable graph. In the world of web programming, Ruby on Rails passes this test; Sinatra and Drupal do not.
+* **petabyte rule**: Wukong/Hanuman coordinate industrial-strength components that wort at terabyte- and petabyte-scale. Conceptual simplicity makes it an excellent tool even for small jobs, but scalability is key. All components must assume an asynchronous, unreliable and distributed system.
+* **laptop rule**:
+* **no dark magick**: the core libraries provide *elegant, predictable magic or no magic at all*. We use metaprogramming heavily, but always predictably, and only in service of making common cases simple. 
+  - Soupy multi-option `case` statements are a smell.
+  - Complex tasks will require code that is more explicit, but readable and organically connected to the typical usage. For example, many data flows will require a custom `Wukong::Streamer` class; but that class is no more complex than the built-in streamer models and receives all the same sugar methods they do.
+* **get shit done**: sometimes ugly tasks require ugly solutions. Shelling out to the hadoop process monitor and parsing its output is acceptable if it is robust and obviates the need for a native protocol handler.
+* **be clever early, boring late**: magic in service of having a terse language for assembling a graph is great. However, the assembled graph should be stomic and largely free of any conditional logic or dependencies.
+  - for example, the data flow `split` statement allows you to set a condition on each branch. The assembled graph, however, is typically a `fanout` stage followed by `filter` stages.
+  - the graph language has some helpers to refer to graph stages. The compiled graph uses explicit mostly-readable but unambiguous static handles.
+  - some stages offer light polymorphism -- for example, `select` accepts either a regexp or block. This is handled at the factory level, and the resulting stage is free of conditional logic.
+* **no lock-in**: needless to say, Wukong works seamlessly with the Infochimps platform, making robust, reliable massive-scale dataflows amazingly simple. However, wukong flows are not tied to the cloud: they project to Hadoop, Flume or any of the other open-source components that power our platform.
+
+__________________________________________________________________________
+
+<a name="stage"></a>
+## Stage
+
+A graph is composed of `stage`s.
+
+* *desc* (alias `description`)
+
+#### Actions
+
+each action 
+
+* the default action is `call`
+* all stages respond to `nothing`, and like ze goggles, do `nothing`.
+
+__________________________________________________________________________
+
+<a name="dataflows"></a>
+## Workflows
+
+Wukong workflows work somewhat differently than you may be familiar with Rake and such.
+
+In wukong, a stage corresponds to a product; you can then act on that product.
+
+Consider first compiling a c program:
+
+    to build the executable, run `cc -o cake eggs.o milk.o flour.o sugar.o -I./include -L./lib`
+    to build files like '{file}.o', run `cc -c -o {file}.o {file}.c -I./include`
+
+In this case, you define the *steps*, implying the products.
+
+
+Something rake can't do (but we should be able to): make it so I can define a dependency that runs **last** 
+
+### Defining jobs
+
+    Wukong.job(:launch) do
+      task :aim do
+        #...
+      end
+      task :enter do
+      end
+      task :commit do
+        # ...
+      end
+    end
+
+    Wukong.job(:recall) do
+      task :smash_with_rock do
+        #...
+      end
+      task :reprogram do
+        # ...
+      end
+    end
+    
+* stages construct products
+  - these have default actions
+* hanuman tracks defined order
+
+* do steps run in order, or is dependency explicit?
+* what about idempotency?
+
+* `task` vs `action` vs `product`; `job`, `task`, `group`, `namespace`.
+
+### documenting
+
+Inline option (`:desc` or `:description`?)
+
+  ```ruby
+      task :foo, :description => "pity the foo" do 
+        # ...
+      end
+  ```
+
+DSL method option
+
+  ```ruby
+      task :foo do
+        description "pity the foo"
+        # ...
+      end
+  ```
+
+### actions
+
+default action:
+
+  ```ruby
+      script 'nukes/launch_codes.rb' do
+        # ...
+      end
+  ```
+  
+define the `:undo` action:
+
+  ```ruby
+      script 'nukes/launch_codes.rb', :undo do
+        # ...
+      end
+  ```      
+
+<a name="file-name-templates"></a>
+### File name templates
+
+* *timestamp*: timestamp of run. everything in this invocation will have the same timestamp.
+* *user*: username; `ENV['USER']` by default
+* *sources*: basenames of job inputs, minus extension, non-`\w` replaced with '_', joined by '-', max 50 chars.
+* *job*:  job flow name
+
+<a name="job-versioning-of-clobbered"></a>
+### versioning of clobbered files
+
+* when files are generated or removed, relocate to a timestamped location
+  - a file `/path/to/file.txt` is relocated to `~/.wukong/backups/path/to/file.txt.wukong-20110102120011` where `20110102120011` is the [job timestamp](#file-naming)
+  - accepts a `max_size` param
+  - raises if it can't write to directory -- must explicitly say `--safe_file_ops=false`
+
+<a name="job-running"></a>
+### running
+
+* `clobber` -- run, but clear all dependencies
+* `undo`    -- 
+* `clean`   -- 
+
+
+### Utility and Filesystem tasks
+
+The primitives correspond heavily with Rake and Chef. However, they extend them in many ways, don't cover all their functionality in many ways, and incompatible in several ways.
+
+### Configuration
+
+
+#### Commandline args
+
+* handled by configliere: `nukes launch --launch_code=GLG20`
+
+* TODO: configliere needs context-specific config vars, so I only get information about the `launch` action in the `nukes` job when I run `nukes launch --help`
+
+
+
+__________________________________________________________________________
+
+<a name="dataflows"></a>
+## Dataflows
+
+
+Data flows 
+
+* you can have a consumer connect to a provider, or vice versa
+  - producer binds to a port, consumers connect to it: pub/sub
+  - consumers open a port, producer connects to many: megaphone
+
+* you can bring the provider on line first, and the consumers later, or vice versa.
+
+
+<a name="dataflow-syntax"></a>
+## Syntax 
+
+**note: this is a scratch pad; actual syntax evolving rapidly and currently looks not much like the following**
+   
+   read('/foo/bar')         # source( FileSource.new('/foo/bar') )
+   writes('/foo/bar')       # sink(   FileSink.new('/foo/bar') )
+
+   ... | file('/foo/bar')   # this we know is a source
+   file('/foo/bar') | ...   # this we know is a sink
+   file('/foo/bar')         # don't know; maybe we can guess later
+
+Here is an example Wukong script, `count_followers.rb`:
+
+    from :json
+    
+    mapper do |user|
+      year_month = Time.parse(user[:created_at]).strftime("%Y%M")
+      emit [ user[:followers_count], year_month ]
+    end   
+    
+    reducer do 
+      start{ @count = 0 }
+    
+      each do |followers_count, year_month|
+        @count += 1
+      end
+    
+      finally{ emit [*@group_key, @count] }
+    end
+    
+You can run this from the commandline:
+
+    wukong count_followers.rb users.json followers_histogram.tsv
+    
+It will run in local mode, effectively doing
+
+    cat users.json | {the map block} | sort | {the reduce block} > followers_histogram.tsv
+
+You can instead run it in Hadoop mode, and it will launch the job across a distributed Hadoop cluster
+
+    wukong --run=hadoop count_followers.rb users.json followers_histogram.tsv
+
+<a name="formatters"></a>
+#### Data Formats (Serialization / Deserialization)
+
+* tsv/csv
+* json
+* xml
+* avro
+* apache_log
+* flat 
+* regexp 
+* [Tagged Netstrings](http://tnetstrings.org/)
+* [ZeroMQ Property Language](http://rfc.zeromq.org/spec:4)
+
+* gz/bz2/zip/snappy 
+
+<a name="data-packets"></a>
+#### Data Packets
+
+Data consists of
+
+- record
+- schema
+- metadata
+
+## Delivery Guarantees
+
+Most messaging systems keep metadata about what messages have been consumed on the broker. That is, as a message is handed out to a consumer, the broker records that fact locally. This is a fairly intuitive choice, and indeed for a single machine server it is not clear where else it could go. Since the data structure used for storage in many messaging systems scale poorly, this is also a pragmatic choice--since the broker knows what is consumed it can immediately delete it, keeping the data size small.
+
+What is perhaps not obvious, is that getting the broker and consumer to come into agreement about what has been consumed is not a trivial problem. If the broker records a message as consumed immediately every time it is handed out over the network, then if the consumer fails to process the message (say because it crashes or the request times out or whatever) then that message will be lost. To solve this problem, many messaging systems add an acknowledgement feature which means that messages are only marked as sent not consumed when they are sent; the broker waits for a specific acknowledgement from the consumer to record the message as consumed. This strategy fixes the problem of losing messages, but creates new problems. First of all, if the consumer processes the message but fails before it can send an acknowledgement then the message will be consumed twice. The second problem is around performance, now the broker must keep multiple states about every single message (first to lock it so it is not given out a second time, and then to mark it as permanently consumed so that it can be removed). Tricky problems must be dealt with, like what to do with messages that are sent but never acknowledged.
+
+So clearly there are multiple possible message delivery guarantees that could be provided:
+
+* At most once—this handles the first case described. Messages are immediately marked as consumed, so they can't be given out twice, but many failure scenarios may lead to losing messages.
+* At least once—this is the second case where we guarantee each message will be delivered at least once, but in failure cases may be delivered twice.
+* Exactly once—this is what people actually want, each message is delivered once and only once.
+
+
+__________________________________________________________________________
+
+<a name="design-questions"></a>
+## Design Questions
+
+* **filename helpers**: 
+  - `':data_dir:/this/that/:script:-:user:-:timestamp:.:ext:'`?
+  - `path_to(:data_dir, 'this/that', "???")`?
+
+* `class Wukong::Foo::Base` vs `class Wukong::Foo` 
+  - the latter is more natural, and still allows 
+  - I'd like 
+
+
+
+__________________________________________________________________________
+__________________________________________________________________________
+__________________________________________________________________________
+
+
+<a name="references"></a>
+## References
+
+<a name="refs-workflow"></a>
+### Workflow
+
+* **Rake**
+
+  - [Rake Docs](http://rdoc.info/gems/rake/file/README.rdoc)
+  - [Rake Tutorial](http://jasonseifer.com/2010/04/06/rake-tutorial) by Jason Seifer -- 2010, with a good overview of why Rake is useful
+  - [Rake Tutorial](http://martinfowler.com/articles/rake.html) by Martin Fowler -- from 2005, so may lack some modernities
+  - [Rake Tutorial](http://onestepback.org/index.cgi/Tech/Rake/Tutorial/RakeTutorialRules.red) -- from 2005, so may lack some modernities
+  
+* **Rake Examples**
+
+  - [resque's redis.rake](https://github.com/defunkt/resque/blob/master/lib/tasks/redis.rake) and [resque/tasks](https://github.com/defunkt/resque/blob/master/lib/resque/tasks.rb)
+  - [rails' Rails Ties](https://github.com/rails/rails/tree/master/railties/lib/rails/tasks)
+  
+* **Thor**
+
+  - [Thor Wiki](https://github.com/wycats/thor/wiki)
+  - 
+  
+* **Chef**
+
+  - [Chef Wiki](http://wiki.opscode.com/display/chef/Home)
+  - specifically, [Chef Resources](http://wiki.opscode.com/display/chef/Resources)
+
+* **Other**
+
+  - [**Gradle**](http://gradle.org/) -- a modern take on `ant` + `maven`. The [Gradle overview](http://gradle.org/overview) states its case.
+
+<a name="refs-dataflow"></a>
+### Dataflow
+
+* **Esper**
+
+  - Must read: [StreamSQL Event Processing with Esper](http://www.igvita.com/2011/05/27/streamsql-event-processing-with-esper/)
+  - [Esper docs](http://esper.codehaus.org/esper-4.5.0/doc/reference/en/html_single/index.html#epl_clauses)
+  - [Esper EPL Reference](http://esper.codehaus.org/esper-4.5.0/doc/reference/en/html_single/index.html#epl_clauses)
+
+* **Storm**
+
+  - [A Storm is coming: more details and plans for release](http://engineering.twitter.com/2011/08/storm-is-coming-more-details-and-plans.html)
+  - [Storm: distributed and fault-tolerant realtime computation](http://www.slideshare.net/nathanmarz/storm-distributed-and-faulttolerant-realtime-computation) -- slideshare presentation
+  - [Storm: the Hadoop of Realtime Processing](http://tech.backtype.com/preview-of-storm-the-hadoop-of-realtime-proce)
+
+* **Kafka**: LinkedIn's high-throughput messaging queue
+
+  - [Kafka's Design: Why we built this](http://incubator.apache.org/kafka/design.html) 
+
+* **ZeroMQ**: tcp sockets like you think they should work
+
+  - [ZeroMQ: A Modern & Fast Networking Stack](http://www.igvita.com/2010/09/03/zeromq-modern-fast-networking-stack/)
+  - [ZeroMQ Guide](http://zguide.zeromq.org/page:all)
+  - [ZeroMQ: An Introduction](http://nichol.as/zeromq-an-introduction)
+  - [Routing with Ruby & ZeroMQ Devices](http://www.igvita.com/2010/11/17/routing-with-ruby-zeromq-devices/)
+  - [Ruby bindings for ZeroMQ](http://zeromq.github.com/rbzmq/) and the [Ruby-FFI bindings](http://www.zeromq.org/bindings:ruby-ffi)
+  - [Learn ruby ZeroMQ](https://github.com/andrewvc/learn-ruby-zeromq) by @andrewvc
+  
+* **Other**
+
+  - [Infopipes: An abstraction for multimedia streamin](http://web.cecs.pdx.edu/~black/publications/Mms062%203rd%20try.pdf) Black et al 2002
+  - [Yahoo Pipes](http://pipes.yahoo.com/pipes/) 
+  - [Yahoo Pipes wikipedia page](http://en.wikipedia.org/wiki/Yahoo_Pipes)
+  - [Streambase](http://www.streambase.com/products/streambasecep/faqs/) -- Why is is so goddamn hard to find out anything real about a project once it gets an enterprise version? Seriously, the consistent fundamental brokenness of enterprise product is astonishing. It's like they take inspiration from shitty major-label band websites but layer a whiteout of [web jargon bullshit](http://www.dack.com/web/bullshit.html) in place of inessential flash animation. Anyway I think Streambase is kinda similar but who the hell can tell.
+  - [Scribe](http://www.cloudera.com/blog/2008/11/02/configuring-and-using-scribe-for-hadoop-log-collection/)
+  - [Splunk Case Study](http://www.igvita.com/2008/10/22/distributed-logging-syslog-ng-splunk/)
+
+<a name="refs-dataflow"></a>
+### Messaging Queue
+
+- [DripDrop](https://github.com/andrewvc/dripdrop) - a message passing library with a unified API abstracting HTTP, zeroMQ and websockets.
+
+
+<a name="refs-dataflow"></a>
+### Data Processing
+
+* **Hadoop**
+
+  - [Hadoop]()
+ 
+  
+* **Spark/Mesos**
+
+  - [Mesos](http://www.mesosproject.org/)