wukong 3.0.0.pre → 3.0.0.pre2

data/docpages/README-performance.textile

@@ -1,90 +0,0 @@
-job_201006200508_0002	NORMAL	flip	parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b	100.00%
-	s3 => hdfs   	bz2     parser, cond_em 	empty (?)
-	201006200508_0002		35mins, 34sec	1	1812031232	         0		12495736645		7240978546		8180472	388863907			parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b
-
-job_201006200508_0003	NORMAL	flip	parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-dupes	100.00%
-	s3 => hdfs  	bz2	parser, cond_em 	duplicate
-	201006200508_0003		15mins, 50sec	1	1812031232	         0		11877866580		7240978546		8180472	383928615			parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-dupes
-
-job_201006200508_0004	NORMAL	flip	parse_twitter_api_requests.rb---/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-from_hdfs_bz2	100.00%
-	hdfs => hdfs 	bz2	parser, cond_em 	empty
-	201006200508_0004		36mins, 56sec	1	          	1812031232		13334645497		7240978546		8180472	395564272			parse_twitter_api_requests.rb---/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-from_hdfs_bz2
-
-job_201006200508_0005	NORMAL	flip	parse_twitter_api_requests.rb---/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-from_hdfs_bz2_no_cond_em	100.00%
-	hdfs => hdfs 	bz2	parser, no_cond_em	--
-	201006200508_0005		35mins, 23sec	1	          	1812031232		13479823318		7240978546		8180472	396757046			parse_twitter_api_requests.rb---/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-from_hdfs_bz2_no_cond_em
-	
-job_201006200508_0006	NORMAL	flip	hdp-stream-flat-/bin/cat-/data/ripd/com.tw/com.twitter/2009111*-/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111	100.00%
-	hdfs => hdfs   	bz2	`which cat`
-	201006200508_0006		 1mins, 10sec 	1	          	1812031232		 7240978549		7240978546		8180472	  8180472			hdp-stream-flat-/bin/cat-/data/ripd/com.tw/com.twitter/2009111*-/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111
-
-job_201006200508_0007	NORMAL	flip	hdp-stream-flat-/bin/cat-s3n://monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111*-/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111-from_s3n	100.00%
-	s3 => hdfs   	bz2	`which cat`
-	201006200508_0007		 1mins, 55sec 	1	1812031232	         0		 7240978549		7240978546		8180472	  8180472			hdp-stream-flat-/bin/cat-s3n://monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111*-/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111-from_s3n
-
-job_201006200508_0008	NORMAL	flip	parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-no_cond_em-no_db	100.00%
-	hdfs => hdfs        flat    parser  no cond_em                      no db
-	201006200508_0008		10mins, 59sec	1	          	7240978549		13545881166		7240978549		8180472	397172723			parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-no_cond_em-no_db
-
-job_201006200508_0015	NORMAL	flip	parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-no_db	100.00%
-	hdfs => hdfs        flat    parser  cond_em on users only           no DB
-	201006200508_0015		23mins, 48sec	1	          	7240978549		13415414554		7240978549		8180472	396101235			parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-no_db
-
-job_201006200508_0016	NORMAL	flip	parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-yes_db-nodupes	100.00%
-	hdfs => hdfs        flat    parser  cond_em on users only - vanished    saving id/sn to DB
-        201006200508_0016		28mins, 7sec        	1	          0	7240978549		 13414285504		  7240978549		 8180472	 396091251			parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-yes_db-dupes
-
-job_201006200508_0017	NORMAL	flip	parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-yes_db-dupes	100.00%
-	hdfs => hdfs        flat    parser  cond_em on users only - duped   	saving id/sn to DB
-        201006200508_0017		11mins, 51sec       	1	          0	7240978549		 12221205449		  7240978549		 8180472	 386114331			parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-yes_db-dupes
-
-===========================================================================
-== Parse
-==
-
-job_201006200508_0018	NORMAL	flip	parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2010056---/data/sn/tw/rawd/parsed/2010056	100.00%
-        201006200508_0018		11hrs, 12mins, 43sec	1	25560337747	          		141729936525		128606199040		14198839	3918844056			parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2010056---/data/sn/tw/rawd/parsed/2010056
-        
-for foo in 0016 0017 0018 ; do echo $foo ; ~/ics/hadoop/chimpmark/bin/elephantscat.rb job_201006200508_$foo ; done 
-cat ~/timings/job/201006200508/*/*.tsv | wu-lign
-
-        job_id	scraped_at          	run_time        	succ?	s3n_in   	hdfs_in  	    file_in	    hdfs_out	   file_out	      map_in	     map_out	map_recs_in	map_recs_out	red_recs_in	red_recs_out	job_name	
-        201006200508_0002		       35mins, 34sec	1	1812031232 	           0	          0	 12495736645	          0	  7240978546	           0	   8180472	 388863907	        0	        0	parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b
-        201006200508_0003		       15mins, 50sec	1	1812031232 	           0	          0	 11877866580	          0	  7240978546	           0	   8180472	 383928615	        0	        0	parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-dupes
-        201006200508_0004		       36mins, 56sec	1	           	  1812031232	          0	 13334645497	          0	  7240978546	           0	   8180472	 395564272	        0	        0	parse_twitter_api_requests.rb---/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-from_hdfs_bz2
-        201006200508_0005		       35mins, 23sec	1	           	  1812031232	          0	 13479823318	          0	  7240978546	           0	   8180472	 396757046	        0	        0	parse_twitter_api_requests.rb---/data/ripd/com.tw/com.twitter/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-from_hdfs_bz2_no_cond_em
-        201006200508_0006		        1mins, 10sec 	1	           	  1812031232	          0	  7240978549	          0	  7240978546	           0	   8180472	   8180472	        0	        0	hdp-stream-flat-/bin/cat-/data/ripd/com.tw/com.twitter/2009111*-/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111
-        201006200508_0007		        1mins, 55sec 	1	1812031232 	           0	          0	  7240978549	          0	  7240978546	           0	   8180472	   8180472	        0	        0	hdp-stream-flat-/bin/cat-s3n://monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2009111*-/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111-from_s3n
-        201006200508_0008		       10mins, 59sec	1	           	  7240978549	          0	 13545881166	          0	  7240978549	           0	   8180472	 397172723	        0	        0	parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-no_cond_em-no_db
-        201006200508_0015		       23mins, 48sec	1	           	  7240978549	          0	 13415414554	          0	  7240978549	           0	   8180472	 396101235	        0	        0	parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-no_db
-        201006200508_0016		        28mins, 7sec	1	           	  7240978549	          0	 13414285504	          0	  7240978549	           0	   8180472	 396091251	        0	        0	parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-yes_db-dupes
-        201006200508_0017		       11mins, 51sec	1	           	  7240978549	          0	 12221205449	          0	  7240978549	           0	   8180472	 386114331	        0	        0	parse_twitter_api_requests.rb---/data/sn/tw/ripd/com.tw/com.twitter-flat/2009111---/data/sn/tw/rawd/parsed/20091110-20091119b-hdfs-flat-cond_em_users-yes_db-dupes
-        201006200508_0018		11hrs, 12mins, 43sec	1	25560337747	           0	          0	141729936525	          0	128606199040	           0	  14198839	3918844056	        0	        0	parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/2010056---/data/sn/tw/rawd/parsed/2010056
-        201006200508_0021		 8hrs, 50mins, 52sec 	1	           	141779023755	62208536220	 24722859867	73825391771	141729936525	189098533358	3918844056	3918844056	155139258	155139258	Unsplicer                                                                                           
-        201006200508_0029		        1mins, 20sec  	1	           	  1763173995	          0	  1762322014	          0	  1762322014	           0	  22764940	  22764940	        0	        0	hdp-stream-flat-/bin/cat-/data/sn/tw/rawd/unspliced/twitter_user-/tmp/foo                           
-        201006200508_0031		  3hrs, 48mins, 6sec  	1	14930014182	           0	          0	 48106164389	          0	113092707367	           0	   8408164	 753481311	        0	        0	parse_twitter_api_requests.rb---s3n//monkeyshines.infochimps.org/data/ripd/com.tw/com.twitter/201004---/data/sn/tw/rawd/parsed/api/201004
-        201006200508_0034		       30mins, 46sec	1	           	  7170990599	 2203578261	  8389754083	 5031160348	  7170990599	  7170990510	 143461243	 143461241	143461241	 67443309	bulk_load_conversation.rb---/data/sn/tw/fixd/objects/a_replies_b---/data/sn/tw/fixd/apeyeye/conversation/a_replies_b_json
-
-Identity mapper         Wukong          `which cat`             pig
-Identity reducer        wukong          `which cat`             pig
-* no skew
-* data/reducer > ram
-
-Do a sort|uniq on 150GB
-
-
-* 1.8 GB bz2, S3 => HDFS                                1m55s
-* 1.8 GB bz2, HDFS => HDFS                              1m10s
-
-TokyoTyrant, 1 node => 4 m1.large (Balancer)            15_000 inserts/sec
-TokyoTyrant, 20 tasks => 4 m1.large (Balancer)           2_000 inserts/sec
-
-===========================================================================
-
-Parse:
-
-hdp-du s3n://monkeyshines.infochimps.org/data/ripd/com.tw/\*/ > /mnt/tmp/ripd_com.tw-du.tsv
-
-
-
-                                                    1050 entries       	   448483502374	       417.7 GB

data/docpages/README-wulign.textile

@@ -1,65 +0,0 @@
----
-layout: default
-title:  mrflip.github.com/wukong - wu-lign utility
-collapse: false
----
-
-h1. wu-lign -- format a tab-separated file as aligned columns
-
-wu-lign will intelligently reformat a tab-separated file into a tab-separated, space aligned file that is still suitable for further processing. For example, given the log-file input
-
-    <pre><code>
-    2009-07-21T21:39:40 day     65536   3.15479 68750   1171316
-    2009-07-21T21:39:45 doing   65536   1.04533 26230   1053956
-    2009-07-21T21:41:53 hapaxlegomenon  65536   0.87574e-05     23707   10051141
-    2009-07-21T21:44:00 concert 500     0.29290 13367   9733414
-    2009-07-21T21:44:29 world   65536   1.09110 32850   200916
-    2009-07-21T21:44:39 world+series    65536   0.49380 9929    7972025
-    2009-07-21T21:44:54 iranelection    65536   2.91775 14592   136342
-    </code></pre>
-
-wu-lign will reformat it to read
-
-    <pre><code>
-    2009-07-21T21:39:40 day                   65536   3.154791234 68750    1171316
-    2009-07-21T21:39:45 doing                 65536   1.045330000 26230    1053956
-    2009-07-21T21:41:53 hapaxlegomenon        65536   0.000008757 23707   10051141
-    2009-07-21T21:44:00 concert                 500   0.292900000 13367    9733414
-    2009-07-21T21:44:29 world                 65536   1.091100000 32850     200916
-    2009-07-21T21:44:39 world+series          65536   0.493800000  9929    7972025
-    2009-07-21T21:44:54 iranelection          65536   2.917750000 14592     136342
-    </code></pre>
-
-The fields are still tab-delimited by exactly one tab -- only spaces are used to pad out fields. You can still use cuttab and friends to manipulate columns.
-
-wu-lign isn't intended to be smart, or correct, or reliable -- only to be useful for previewing and organizing tab-formatted files. In general @wu-lign(foo).split("\t").map(&:strip)@ *should* give output semantically equivalent to its input. (That is, the only changes should be insertion of spaces and re-formatting of numerics.) But still -- reserve its use for human inspection only.
-
-(Note: tab characters in this source code file have been converted to spaces; replace whitespace with tab in the first example if you'd like to play along at home.)
-
-h2. How it works 
-
-Wu-Lign takes the first 500ish lines, splits into fields on TAB characters, and tries to guess the format (int, float, or string) for each. It builds a consensus of the width and type for corresponding columns in the chunk.  If a column has mixed numeric and string formats it degrades to :mixed, which is basically treated as :string. If a column has mixed :float and :int elements all of them are formatted as float.
-
-h2. Command-line arguments
-
-You can give sprintf-style positional arguments on the command line that will be applied to the corresponding columns. (Blank args are used for placeholding and auto-formatting is still applied).  So with the example above,
-
-    @cat foo | wu-lign  '' '' '' '%8.4e'@
-
-will format the fourth column with "%8.4e", while the first three columns and fifth-and-higher columns are formatted as usual.
-
-    <pre><code>
-    ...
-    2009-07-21T21:39:45 doing           65536   1.0453e+00      26230    1053956
-    2009-07-21T21:41:53 hapaxlegomenon  65536   8.7574e-06      23707   10051141
-    2009-07-21T21:44:00 concert           500   2.9290e-01      13367    9733414
-    ....
-    </code></pre>
-
-h2. Notes
-
-* Header rows: the first line is used for width alignment but not for type detection.  This means that an initial row of text headers will inform column spacing but still allow a column of floats (say) to be properly aligned as floats.
-* It requires a unanimous vote. One screwy line can coerce the whole mess to :mixed; width formatting will still be applied, though.
-* It won't set columns wider than 100 chars -- this allows for the occasional super-wide column without completely breaking your screen.
-* For :float values, wulign tries to guess at the right number of significant digits to the left and right of the decimal point.
-* wulign parses only plain-jane 'TSV files': no quoting or escaping; every tab delimits a field, every newline a record.

data/docpages/UsingWukong-part1-get_ready.textile

@@ -1,17 +0,0 @@
----
-layout: default
-title:  mrflip.github.com/wukong - Using Wukong and Wuclan, Part 1 - Setup
-collapse: false
----
-
-h1. Using Wukong and Wuclan, Part 0 - Setup
-
-Please follow the "installation and setup directions":setup.html for wukong, hadoop and a compute cluster.
-
-h1. Using Wukong and Wuclan, Part 1 - Scraping
-
-This part needs writing.
-
-Later, it will tell you how to get a large corpus of data to use in part 2.
-
-In the meantime check out http://mrflip.github.com/monkeyshines/ and http://mrflip.github.com/wuclan/ -- in particular the "Twitter Search Scraper":http://github.com/mrflip/wuclan/tree/master/examples/twitter/scrape_twitter_search/ example.  We use this in production to gather and analyze tens of gigabytes of twitter conversations.

data/docpages/UsingWukong-part2-ThinkingBigData.textile

@@ -1,75 +0,0 @@
----
-layout: default
-title:  mrflip.github.com/wukong - Overview
-collapse: false
----
-
-h1. Thinking Big Data
-
-h2. There's lots of data, Wukong and Hadoop can help
-
-
-There are two disruptive
-
-* We're instrumenting every realm of human activity
-** Conversation
-** Relationships
-** 
-
-* We have linearly scaling multiprocessing
-** Old frontier computing: expensive, N log N, SUUUUUUCKS
-** It's cheap, it's scaleable and it's fun
-
-h2. == Map|Reduce ==
-
-h3. cat input.tsv | mapper.sh | sort | reducer.sh > output.tsv
-
-* Bobo histogram:
-
-  cat twitter_users.tsv | cuttab 3 | cutc 1-6 | sort | uniq -c > histogram.tsv
-
-  cat twitter_users.tsv | \
-    cuttab 3 |                  # extract the date column               \
-    cutc 1-6 |                  # chop off all but the yearmonth        \
-    sort     |                  # sort, to ensure locality              \
-    uniq -c  >                  # roll up lines, along with their count \
-    histogram.tsv               # save into output file
-  
-  
-h3. Word Count
-
-mapper:
-
-    # output each word on its own line
-    @readlines.each{|line| puts line.split(/[^\w]+/) }@
-
-reducer:
-
-    # every word is _guaranteed_ to land in the same place and next to its
-    # friends, so we can just output the repetition count for each
-    # distinct line.
-    uniq -c
-
-
-h3. Word Count by Person
-
-* Partition Keys vs. Reduce Keys
-
-- reduce by [word, <total>, count] and [word, user_id, count]
-
-
-h2. == Global Structure ==
-
-h3. Enumerating neighborhood
-
-* adjacency list
-
-* join on center link
-
-* list of 3-paths == 
-
-h2. == Mechanics, HDFS ==
-
-
-x M _
-_ M y

data/docpages/UsingWukong-part3-parsing.textile

@@ -1,138 +0,0 @@
----
-layout: default
-title:  mrflip.github.com/wukong - Using Wukong and Wuclan, Part 3 - Parsing
-collapse: false
----
-
-h1. Using Wukong and Wuclan - Parsing
-
-In part 1 we begain a scraper to trawl our desired part of the social web. Now
-we're ready to start using Wukong to process the files.
-
-Files come off the wire as
-
-    :url	:scraped_at	:response_code	:response_message	:contents
-    String      DateTime (flat) Integer          String                 String (JSON-formatted, tab&newline-munged)
-
-The contents field is a JSON-formatted mix of records:
-
-* TwitterFollowersRequest and TwitterFriendsRequest yield an @Array[Hash{user => raw_tweet}]@. We want to extract a stream of AFollowsB (with the request user as user_a for a friends request and user_b for a followers request) along with the included Tweet, TwitterUser, TwitterUserProfile and TwitterUserStyle records.
-* TwitterFavoritesRequest yields an array of @Array[Hash{tweet_hash => user_hash}]. We want to extract a stream of AFavoritesB along with the included Tweet, TwitterUser, TwitterUserProfile and TwitterUserStyle records
-* TwitterUser yields a single @user_hash@ making one each of TwitterUser, TwitterUserProfile and TwitterUserStyle.
-* UserTimelineRequest and PublicTimelineRequest yield an Array[Hash{tweet => user}]. We want to extract the included Tweet, TwitterUser, TwitterUserProfile and TwitterUserStyle records.
-* TwitterFollowerIdsRequest and TwitterFriendIdsRequest return an Array[user_ids] (each user_id is a simple Integer). We extract a series of AFollowsB (using the request's user_id as user_a_id or user_b_id)
-
-We want to split each API response into a stream of those TwitterUser, Tweet, etc. records.
-
-# Stream in each line (each line holds one request)
-# turn the line into the corresponding TwitterRequest
-# have the TwitterRequest parse its JSON contents and construct the TwitterUser, Tweet, etc.
-# seriealize those records back out as tab-separated lines suitable for further processing with Wukong
-
-h4. The basics of StructStreamer
-
-Wukong handles the first and last steps through its StructStreamer and the standard .to_flat method. So the actual structure is really simple:
-
-        #
-        # Instantiate each incoming request.
-        # Stream out the contained classes it generates.
-        #
-        class TwitterRequestParser < Wukong::Streamer::StructStreamer
-          def process request
-            request.parse do |obj|
-              yield obj
-            end
-          end
-        end
-
-        # This makes the script go.
-        Wukong::Script.new(TwitterRequestParser, nil).run
-
-In practice, all you need to know is that a StructStreamer gets a stream of objects to parse.  Here's an outline of its internals. The Wukong StructStreamer:
-
-# takes each flattened line:
-
-    "twitter_friends_request	http://.... 	20090701123456	...fields...	[{...}, {...}, ...json..., {...}]"
-
-# splits by tabs to create an array of fields
-
-    ["twitter_friends_request", "http://...", ... "[{...}, {...}, ...json..., {...}]"]
-
-# constructs the class name indicated in the first field,
-  using the values extracted from the remaining fields.
-
-    TwitterFriendsRequest.new "http://...", "20090701123456", ... "[{...}, {...}, ...json..., {...}]"
-
-The last (contents) field is still just a string: there's nothing special about it to Wukong.
-
-h4. Parsing
-
-Since each requests' contents are handled in a slightly (and brittle-ly) different manner, we just ask each request object to parse itself and feed out all the TwitterXXXX objects it generates.
-
-    class TwitterFollowersRequest
-        # ...
-
-        def parse &block
-          return unless healthy?
-          # for each raw user/tweet pair in the parsed JSON contents,
-          parsed_contents.each do |hsh|
-            json_obj = JsonUserWithTweet.new(hsh, 'scraped_at' => scraped_at)
-            next unless json_obj && json_obj.healthy?
-            # Extract user, tweet and relationship
-            yield AFollowsB.new(json_obj.user.id, self.twitter_user_id) if json_obj.user
-            json_obj.each(&block)
-          end
-        end
-        
-        # ...
-     end
-
-The TwitterXXXRequest objects consist of one or many hashes with (a raw user hash, and possibly its latest raw tweet hash) or (a raw tweet hash and its raw user hash).  The user hash might have only the fields for a TwitterPartialUser or it might have the fields for a full set of TwitterUser, TwitterUserProfile, TwitterUserStyle. Besides which, the fields themselves need some massaging to be compatible with Wukong and other tools in our Map/Reduce toolkit (details explained in a later section).
-
-The fiddly little details are handled by a JsonUserWithTweet or JsonTweetWithUser (as appropriate) adapter pattern:
-
-    class JsonUserTweetPair
-      def initialize raw, moreinfo
-        # clean up fields in entries (flatten date, true/false -> 1/0, etc)
-        fix_raw_user!
-        fix_raw_tweet!
-      end
-
-      # generate all the contained TwitterXXX objects
-      def each
-        # 
-      end
-
-      # create TwitterUser object from raw info
-      def user
-      end
-      # create Tweet object from raw tweet hash
-      def tweet
-      end
-      # ... and so forth
-    end
-      
-I'll ignore the gory details; view the source if you're interested.
-
-
-h4. Running the script
-
-Here, again, is the code (in full!) for the twitter_request_parser.rb script.
-
-    #
-    # Instantiate each incoming request.
-    # Stream out the contained classes it generates.
-    #
-    class TwitterRequestParser < Wukong::Streamer::StructStreamer
-      def process request
-        request.parse do |obj|
-          yield obj
-        end
-      end
-    end
-
-    # This makes the script go.
-    Wukong::Script.new(TwitterRequestParser, nil).run
-
-That last line is the runner: it makes this a Wukong script with a map phase only.  (We'll add in a reducer later on.)
-

data/docpages/_config.yml

@@ -1,39 +0,0 @@
----
-permalink:      ":year-:month/:title.html"
-markdown:       rdiscount
-pygments:       true
-auto:           true
-server:         true
-server_port:    4000
-maruku:
-  use_tex:      false
-  use_divs:     false
-  png_dir:      images/latex
-  png_url:      /images/latex
-
-header_ref:     '.html'    # .html for subdirs, / for main.
-assets_path:    '/'        # http://github.mrflip.com
-
-gemuser:        mrflip
-gemname:        wukong
-gemversion:     0.1.1
-title:          mrflip.github.com/wukong
-
-keywords:       [ 'wukong,hadoop,ruby,mrflip,infochimps,map,reduce,streaming,dumbo,happy,mrtoolkit,script,simple' ]
-description:    "Wukong: Hadoop made so easy a Chimpanzee could run it."
-header_files:
-  - INSTALL
-  - LICENSE
-  - usage
-  - wutils
-  - moreinfo
-  - tutorial
-
-credits:
-  <p>Wukong image courtesy
-  <a href="http://www.curtbusse.com/okavango/page1/oka1.html">Curt Busse</a> under
-  an <a href="http://www.curtbusse.com/copyright.html">open license</a>.
-  It's a Chacma Baboon from the Okavango site. Make sure to read the
-  <a href="http://www.curtbusse.com/okavango/page1/oka1.html#note3">story at the bottom of that page</a>.
-  </p>
-

data/docpages/avro/avro_notes.textile

@@ -1,56 +0,0 @@
-* Spec: http://avro.apache.org/docs/current/spec.html
-* Jira: https://issues.apache.org/jira/browse/AVRO
-* Wiki: https://cwiki.apache.org/confluence/display/AVRO/Index
-
-* http://github.com/phunt/avro-rpc-quickstart
-
-* http://lucene.apache.org/java/2_4_0/fileformats.html#VInt -- types
-* http://code.google.com/apis/protocolbuffers/docs/encoding.html#types -- a good reference
-* Avro + Eventlet (Python evented code): http://unethicalblogger.com/node/282
-
-
-
-Cassandra + Avro
-
-* Make bulk loading into Cassandra less crappy, more pluggable https://issues.apache.org/jira/browse/CASSANDRA-1278
-* Refactor Streaming: https://issues.apache.org/jira/browse/CASSANDRA-1189
-* Increment Counters: https://issues.apache.org/jira/browse/CASSANDRA-1072
-
-== From hammer's avro tools:
-
-#! /usr/bin/env python
-
-import sys
-from avro import schema
-from avro.genericio import DatumReader
-from avro.io import DataFileReader
-
-if __name__ == "__main__":
-  if len(sys.argv) < 2:
-    print "Need to at least specify an Avro file."
-  outfile_name = sys.argv[1]
-
-  message_schema = None
-  if len(sys.argv) > 2:
-    message_schema = schema.parse(schema.parse(sys.argv[2].encode("utf-8")))
-
-  r = file(outfile_name, 'r')
-  dr = DatumReader(expected = message_schema)
-  dfr = DataFileReader(r, dr)
-  for record in dfr:
-    print record
-  dfr.close()
-
-from binascii import hexlify
-
-def avro_hexlify(reader):
-  """Return the hex value, as a string, of a binary-encoded int or long."""
-  bytes = []
-  current_byte = reader.read(1)
-  bytes.append(hexlify(current_byte))
-  while (ord(current_byte) & 0x80) != 0:
-    current_byte = reader.read(1)
-    bytes.append(hexlify(current_byte))
-  return ' '.join(bytes)
-
-