mrflip-wukong 0.1.0

data/bin/tabchar

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# insert a tab char from the command line:
+# echo "hi$(tabchar)there"
+# # => "hi	there"
+echo -n -e '\t'

data/bin/uniqc

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+uniq -c | ruby -ne 'puts $_.chomp.gsub(/^\s+(\d+)\s+/){ "%15s\t" % $1 }'

data/bin/wu-hist

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+sort | uniq -c | sort -rn | ruby -ne 'puts $_.chomp.gsub(/^\s+(\d+)\s+/){ $1+"\t" }'

data/bin/wu-lign

@@ -0,0 +1,177 @@
+#!/usr/bin/env ruby
+
+USAGE= %Q{
+# h1. wulign -- format a tab-separated file as aligned columns
+#
+# wulign will intelligently reformat a tab-separated file into a tab-separated,
+# space aligned file that is still suitable for further processing. For example,
+# given the log-file input
+#
+#     <pre><code>
+#     2009-07-21T21:39:40 day     65536   3.15479 68750   1171316
+#     2009-07-21T21:39:45 doing   65536   1.04533 26230   1053956
+#     2009-07-21T21:41:53 hapaxlegomenon  65536   0.87574e-05     23707   10051141
+#     2009-07-21T21:44:00 concert 500     0.29290 13367   9733414
+#     2009-07-21T21:44:29 world   65536   1.09110 32850   200916
+#     2009-07-21T21:44:39 world+series    65536   0.49380 9929    7972025
+#     2009-07-21T21:44:54 iranelection    65536   2.91775 14592   136342
+#     </code></pre>
+#
+# wulign will reformat it to read
+#
+#     <pre><code>
+#     2009-07-21T21:39:40 day                   65536   3.154791234 68750    1171316
+#     2009-07-21T21:39:45 doing                 65536   1.045330000 26230    1053956
+#     2009-07-21T21:41:53 hapaxlegomenon        65536   0.000008757 23707   10051141
+#     2009-07-21T21:44:00 concert                 500   0.292900000 13367    9733414
+#     2009-07-21T21:44:29 world                 65536   1.091100000 32850     200916
+#     2009-07-21T21:44:39 world+series          65536   0.493800000  9929    7972025
+#     2009-07-21T21:44:54 iranelection          65536   2.917750000 14592     136342
+#     </code></pre>
+#
+# The fields are still tab-delimited by exactly one tab -- only spaces are used to
+# pad out fields. You can still use cuttab and friends to manipulate columns.
+#
+# wulign isn't intended to be smart, or correct, or reliable -- only to be
+# useful for previewing and organizing tab-formatted files. In general
+# @wulign(foo).split("\t").map(&:strip)@ *should* give output semantically
+# equivalent to its input. (That is, the only changes should be insertion of
+# spaces and re-formatting of numerics.) But still -- reserve its use for human
+# inspection only.
+#
+# (Note: tab characters in this source code file have been converted to spaces;
+# replace whitespace with tab in the first example if you'd like to play along at
+# home.)
+#
+# h2. How it works
+#
+# Wulign takes the first 1000 lines, splits by TAB characters into fields, and
+# tries to guess the format -- int, float, or string -- for each. It builds a
+# consensus of the width and type for corresponding columns in the chunk.  If a
+# column has mixed numeric and string formats it degrades to :mixed, which is
+# basically treated as :string. If a column has mixed :float and :int elements all
+# of them are formatted as float.
+#
+# h2. Command-line arguments
+#
+# You can give sprintf-style positional arguments on the command line that will be
+# applied to the corresponding columns. (Blank args are used for placeholding and
+# auto-formatting is still applied).  So with the example above,
+#
+#     @cat foo | wulign  '' '' '' '%8.4e'@
+#
+# will format the fourth column with "%8.4e", while the first three columns and
+# fifth-and-higher columns are formatted as usual.
+#
+#     <pre><code>
+#     ...
+#     2009-07-21T21:39:45 doing           65536   1.0453e+00      26230    1053956
+#     2009-07-21T21:41:53 hapaxlegomenon  65536   8.7574e-06      23707   10051141
+#     2009-07-21T21:44:00 concert           500   2.9290e-01      13367    9733414
+#     ....
+#     </code></pre>
+#
+# h2. Notes
+#
+# * It has no knowledge of header rows. An all-text first line will screw everything up.
+#
+# * It also requires a unanimous vote. One screwy line can coerce the whole mess
+#   to :mixed; width formatting will still be applied, though.
+#
+# * It won't set columns wider than 70 chars -- this allows for the occasional
+#   super-wide column without completely breaking your screen.
+#
+# * For :float values, wulign tries to guess at the right number of significant
+#   digits to the left and right of the decimal point.
+#
+# * wulign does not parse 'TSV files' in their strict sense -- there is no quoting
+#   or escaping; every tab delimits a field, every newline a record.
+}
+
+if ARGV[0] == '--help'
+  puts $0
+  puts USAGE
+  exit
+end
+
+#
+# How many initial lines to use to guess formatting.  Lines after this are
+# simply reformatted according to the consensus of the initial
+# FORMAT_GUESSING_LINES.
+#
+FORMAT_GUESSING_LINES = 500
+# widest column to set
+MAX_MAX_WIDTH = 70
+
+INT_RE   = /\A\d+\z/
+FLOAT_RE = /\A(\d+)(?:\.(\d+))?(?:e-?\d+)?\z/
+
+def consensus_type val, alltype
+  return :mixed if alltype == :mixed
+  case
+  when val == ''       then type = nil
+  when val =~ INT_RE   then type = :int
+  when val =~ FLOAT_RE then type = :float
+  else                      type = :str end
+  return if ! type
+  case
+  when alltype.nil?    then type
+  when alltype == type then type
+  when ( ((alltype==:float) && (type == :int)) || ((alltype == :int) && (type == :float)) )
+    :float
+  else :mixed
+  end
+end
+
+def f_width str
+  str =~ FLOAT_RE or return 0
+  [$1.length, $2 ? $2.length : 0]
+end
+
+maxw       = []
+col_types  = []
+col_minmag = []
+col_maxmag = []
+rows       = []
+skip_col   = []
+ARGV.each_with_index{|v,i| next if (v == '') ; maxw[i] = 0; skip_col[i] = true }
+FORMAT_GUESSING_LINES.times do
+  line = $stdin.readline rescue nil
+  break unless line
+  cols = line.chomp.split("\t").map{|s| s.strip }
+  col_widths = cols.map{|col| col.length }
+  col_widths.each_with_index{|cw,i| maxw[i] = [[cw,maxw[i]].compact.max, MAX_MAX_WIDTH].min }
+  cols.each_with_index{|col,i|
+    next if skip_col[i]
+    col_types[i] = consensus_type(col, col_types[i])
+    if col_types[i] == :float
+      mantissa, radix = f_width(col)
+      col_minmag[i] = [radix,    col_minmag[i], 1].compact.max
+      col_maxmag[i] = [mantissa, col_maxmag[i], 1].compact.max
+    end
+  }
+  # p [maxw, col_types, col_minmag, col_maxmag, col_widths, cols]
+  rows << cols
+end
+
+format = maxw.zip(col_types, col_minmag, col_maxmag, ARGV).map do |width, type, minmag, maxmag, default|
+  next(lambda{|s| default % s rescue s }) if default.to_s != ''
+  case type
+  when :mixed, nil then lambda{|s| "%-#{width}s" % s }
+  when :str        then lambda{|s| "%-#{width}s" % s }
+  when :int        then lambda{|s| "%#{width}d"  % s.to_i }
+  when :float      then lambda{|s| "%#{maxmag+minmag+1}.#{minmag}f" % s.to_f }
+  else raise "oops type #{type}"  end
+end
+# p [maxw, col_types, col_minmag, col_maxmag, format]
+
+pad = [''] * maxw.length
+rows.each do |row|
+  # note -- strips trailing columns
+  puts row.zip(format).map{|c,f| f.call(c) }.join("\t")
+end
+$stdin.each do |line|
+  cols = line.chomp.split("\t").map{|s| s.strip }
+  # note -- strips trailing columns
+  puts cols.zip(format).map{|c,f| f.call(c) rescue c }.join("\t")
+end

data/bin/wu-sum

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+require 'wukong'
+require 'wukong/streamer/summing_reducer'
+
+#
+#
+class Summer < Wukong::Streamer::SummingReducer
+  attr_accessor :sample_line
+
+  def initialize *args
+    self.summing_elements = [0]
+    super *args
+  end
+
+  def start! *args
+    self.sample_line = args
+    super *args
+  end
+
+  def get_key *fields
+    fields.values_at(2,3)
+  end
+
+  def finalize
+    summing_elements.each{|idx| sample_line[idx] = sums[idx]}
+    yield sample_line
+  end
+end
+
+Wukong::Script.new(Summer, nil).run

data/doc/README-wulign.textile

@@ -0,0 +1,59 @@
+h1. wulign -- format a tab-separated file as aligned columns
+
+wulign will intelligently reformat a tab-separated file into a tab-separated, space aligned file that is still suitable for further processing. For example, given the log-file input
+
+    <pre><code>
+    2009-07-21T21:39:40 day     65536   3.15479 68750   1171316
+    2009-07-21T21:39:45 doing   65536   1.04533 26230   1053956
+    2009-07-21T21:41:53 hapaxlegomenon  65536   0.87574e-05     23707   10051141
+    2009-07-21T21:44:00 concert 500     0.29290 13367   9733414
+    2009-07-21T21:44:29 world   65536   1.09110 32850   200916
+    2009-07-21T21:44:39 world+series    65536   0.49380 9929    7972025
+    2009-07-21T21:44:54 iranelection    65536   2.91775 14592   136342
+    </code></pre>
+
+wulign will reformat it to read
+
+    <pre><code>
+    2009-07-21T21:39:40 day                   65536   3.154791234 68750    1171316
+    2009-07-21T21:39:45 doing                 65536   1.045330000 26230    1053956
+    2009-07-21T21:41:53 hapaxlegomenon        65536   0.000008757 23707   10051141
+    2009-07-21T21:44:00 concert                 500   0.292900000 13367    9733414
+    2009-07-21T21:44:29 world                 65536   1.091100000 32850     200916
+    2009-07-21T21:44:39 world+series          65536   0.493800000  9929    7972025
+    2009-07-21T21:44:54 iranelection          65536   2.917750000 14592     136342
+    </code></pre>
+
+The fields are still tab-delimited by exactly one tab -- only spaces are used to pad out fields. You can still use cuttab and friends to manipulate columns.
+
+wulign isn't intended to be smart, or correct, or reliable -- only to be useful for previewing and organizing tab-formatted files. In general @wulign(foo).split("\t").map(&:strip)@ *should* give output semantically equivalent to its input. (That is, the only changes should be insertion of spaces and re-formatting of numerics.) But still -- reserve its use for human inspection only.
+
+(Note: tab characters in this source code file have been converted to spaces; replace whitespace with tab in the first example if you'd like to play along at home.)
+
+h2. How it works 
+
+Wulign takes the first 1000 lines, splits by TAB characters into fields, and tries to guess the format -- int, float, or string -- for each. It builds a consensus of the width and type for corresponding columns in the chunk.  If a column has mixed numeric and string formats it degrades to :mixed, which is basically treated as :string. If a column has mixed :float and :int elements all of them are formatted as float.
+
+h2. Command-line arguments
+
+You can give sprintf-style positional arguments on the command line that will be applied to the corresponding columns. (Blank args are used for placeholding and auto-formatting is still applied).  So with the example above,
+
+    @cat foo | wulign  '' '' '' '%8.4e'@
+
+will format the fourth column with "%8.4e", while the first three columns and fifth-and-higher columns are formatted as usual.
+
+    <pre><code>
+    ...
+    2009-07-21T21:39:45 doing           65536   1.0453e+00      26230    1053956
+    2009-07-21T21:41:53 hapaxlegomenon  65536   8.7574e-06      23707   10051141
+    2009-07-21T21:44:00 concert           500   2.9290e-01      13367    9733414
+    ....
+    </code></pre>
+
+h2. Notes
+
+* It has no knowledge of header rows. An all-text first line will screw everything up.
+* It also requires a unanimous vote. One screwy line can coerce the whole mess to :mixed; width formatting will still be applied, though.
+* It won't set columns wider than 70 chars -- this allows for the occasional super-wide column without completely breaking your screen.
+* For :float values, wulign tries to guess at the right number of significant digits to the left and right of the decimal point.
+* wulign does not parse 'TSV files' in their strict sense -- there is no quoting or escaping; every tab delimits a field, every newline a record.

data/doc/README-wutils.textile

@@ -0,0 +1,128 @@
+h1. Wukong Utility Scripts
+
+h2. Stupid command-line tricks
+
+h3. Histogram
+
+Given data with a date column:
+
+   message	235623	20090423012345	Now is the winter of our discontent Made glorious summer by this son of York
+   message	235623	20080101230900	These pretzels are making me THIRSTY!
+   ...
+
+You can calculate number of messages sent by day with
+
+    cat messages | cuttab 3 | cutc 8 | sort | uniq -c
+
+(see the wuhist command, below.)
+
+h3. Simple intersection, union, etc
+
+For two datasets (batch_1 and batch_2) with unique entries (no repeated lines),
+
+* Their union is simple:
+
+      cat batch_1 batch_2 | sort -u
+
+
+* Their intersection:
+
+      cat batch_1 batch_2 | sort | uniq -c | egrep -v '^ *1 '
+
+  This concatenates the two sets and filters out everything that only occurred once.
+
+* For the complement of the intersection, use "... | egrep '^ *1 '"
+  
+* In both cases, if the files are each internally sorted, the commandline sort takes a --merge flag:
+
+      sort --merge -u batch_1 batch_2 
+
+h2. Command Listing
+
+h3. cutc
+
+@cutc [colnum]@
+
+Ex.
+
+  echo -e 'foo\tbar\tbaz' | cutc 6
+  foo	ba
+
+Cuts from beginning of line to given column (default 200). A tab is one character, so right margin can still be ragged.
+ 
+h3. cuttab
+
+  @cuttab [colspec]@
+
+Cuts given tab-separated columns. You can give a comma separated list of numbers
+or ranges 1-4. columns are numbered from 1.
+
+Ex.
+
+  echo -e 'foo\tbar\tbaz' | cuttab 1,3
+  foo	baz
+
+h3. hdp-*
+
+These perform the corresponding commands on the HDFS filesystem.  In general,
+where they accept command-line flags, they go with the GNU-style ones, not the
+hadoop-style: so, @hdp-du -s dir@ or @hdp-rm -r foo/@
+
+* @hdp-cat@
+* @hdp-catd@ -- cats the files that don't start with '_' in a directory. Use this for a pile of @.../part-00000@ files
+* @hdp-du@
+* @hdp-get@
+* @hdp-kill@
+* @hdp-ls@
+* @hdp-mkdir@
+* @hdp-mv@
+* @hdp-ps@
+* @hdp-put@
+* @hdp-rm@
+* @hdp-sync@
+
+h3. hdp-sort, hdp-stream, hdp-stream-flat
+
+* @hdp-sort@
+* @hdp-stream@
+* @hdp-stream-flat@
+
+    <code><pre>
+    hdp-stream input_filespec output_file map_cmd reduce_cmd num_key_fields
+    </pre></code>
+
+h3. tabchar
+
+Outputs a single tab character.
+ 
+h3. wuhist
+
+Occasionally useful to gather a lexical histogram of a single column:
+
+Ex.
+
+    <code><pre>
+    $ echo -e 'foo\nbar\nbar\nfoo\nfoo\nfoo\n7' | ./wuhist
+    4       foo
+    2       bar
+    1       7
+    </pre></code>
+
+(the output will have a tab between the first and second column, for futher processing.)
+
+h3. wulign
+
+Intelligently format a tab-separated file into aligned columns (while remaining tab-separated for further processing). See README-wulign.textile.
+ 
+h3. hdp-parts_to_keys.rb
+
+A *very* clumsy script to rename reduced hadoop output files by their initial key.
+
+If your output file has an initial key in the first column and you pass it
+through hdp-sort, they will be distributed across reducers and thus output
+files. (Because of the way hadoop hashes the keys, there's no guarantee that
+each file will get a distinct key. You could have 2 keys with a million entries
+and they could land sequentially on the same reducer, always fun.)
+
+If you're willing to roll the dice, this script will rename files according to
+the first key in the first line.

data/doc/UsingWukong-part1.textile

@@ -0,0 +1,2 @@
+h1. Using Wukong and Wuclan, Part 1 - Setup
+

data/doc/UsingWukong-part2.textile

@@ -0,0 +1,2 @@
+h1. Using Wukong and Wuclan, Part 2 - Scraping
+

data/doc/UsingWukong-part3-parsing.textile

@@ -0,0 +1,132 @@
+h1. Using Wukong and Wuclan, Part 3 - Parsing
+
+In part 2 we begain a scraper to trawl our desired part of the social web. Now
+we're ready to start using Wukong to process the files.
+
+Files come off the wire as
+
+    :url	:scraped_at	:response_code	:response_message	:contents
+    String      DateTime (flat) Integer          String                 String (JSON-formatted, tab&newline-munged)
+
+The contents field is a JSON-formatted mix of records:
+
+* TwitterFollowersRequest and TwitterFriendsRequest yield an @Array[Hash{user => raw_tweet}]@. We want to extract a stream of AFollowsB (with the request user as user_a for a friends request and user_b for a followers request) along with the included Tweet, TwitterUser, TwitterUserProfile and TwitterUserStyle records.
+* TwitterFavoritesRequest yields an array of @Array[Hash{tweet_hash => user_hash}]. We want to extract a stream of AFavoritesB along with the included Tweet, TwitterUser, TwitterUserProfile and TwitterUserStyle records
+* TwitterUser yields a single @user_hash@ making one each of TwitterUser, TwitterUserProfile and TwitterUserStyle.
+* UserTimelineRequest and PublicTimelineRequest yield an Array[Hash{tweet => user}]. We want to extract the included Tweet, TwitterUser, TwitterUserProfile and TwitterUserStyle records.
+* TwitterFollowerIdsRequest and TwitterFriendIdsRequest return an Array[user_ids] (each user_id is a simple Integer). We extract a series of AFollowsB (using the request's user_id as user_a_id or user_b_id)
+
+We want to split each API response into a stream of those TwitterUser, Tweet, etc. records.
+
+# Stream in each line (each line holds one request)
+# turn the line into the corresponding TwitterRequest
+# have the TwitterRequest parse its JSON contents and construct the TwitterUser, Tweet, etc.
+# seriealize those records back out as tab-separated lines suitable for further processing with Wukong
+
+h4. The basics of StructStreamer
+
+Wukong handles the first and last steps through its StructStreamer and the standard .to_flat method. So the actual structure is really simple:
+
+        #
+        # Instantiate each incoming request.
+        # Stream out the contained classes it generates.
+        #
+        class TwitterRequestParser < Wukong::Streamer::StructStreamer
+          def process request
+            request.parse do |obj|
+              yield obj
+            end
+          end
+        end
+
+        # This makes the script go.
+        Wukong::Script.new(TwitterRequestParser, nil).run
+
+In practice, all you need to know is that a StructStreamer gets a stream of objects to parse.  Here's an outline of its internals. The Wukong StructStreamer:
+
+# takes each flattened line:
+
+    "twitter_friends_request	http://.... 	20090701123456	...fields...	[{...}, {...}, ...json..., {...}]"
+
+# splits by tabs to create an array of fields
+
+    ["twitter_friends_request", "http://...", ... "[{...}, {...}, ...json..., {...}]"]
+
+# constructs the class name indicated in the first field,
+  using the values extracted from the remaining fields.
+
+    TwitterFriendsRequest.new "http://...", "20090701123456", ... "[{...}, {...}, ...json..., {...}]"
+
+The last (contents) field is still just a string: there's nothing special about it to Wukong.
+
+h4. Parsing
+
+Since each requests' contents are handled in a slightly (and brittle-ly) different manner, we just ask each request object to parse itself and feed out all the TwitterXXXX objects it generates.
+
+    class TwitterFollowersRequest
+        # ...
+
+        def parse &block
+          return unless healthy?
+          # for each raw user/tweet pair in the parsed JSON contents,
+          parsed_contents.each do |hsh|
+            json_obj = JsonUserWithTweet.new(hsh, 'scraped_at' => scraped_at)
+            next unless json_obj && json_obj.healthy?
+            # Extract user, tweet and relationship
+            yield AFollowsB.new(json_obj.user.id, self.twitter_user_id) if json_obj.user
+            json_obj.each(&block)
+          end
+        end
+        
+        # ...
+     end
+
+The TwitterXXXRequest objects consist of one or many hashes with (a raw user hash, and possibly its latest raw tweet hash) or (a raw tweet hash and its raw user hash).  The user hash might have only the fields for a TwitterPartialUser or it might have the fields for a full set of TwitterUser, TwitterUserProfile, TwitterUserStyle. Besides which, the fields themselves need some massaging to be compatible with Wukong and other tools in our Map/Reduce toolkit (details explained in a later section).
+
+The fiddly little details are handled by a JsonUserWithTweet or JsonTweetWithUser (as appropriate) adapter pattern:
+
+    class JsonUserTweetPair
+      def initialize raw, moreinfo
+        # clean up fields in entries (flatten date, true/false -> 1/0, etc)
+        fix_raw_user!
+        fix_raw_tweet!
+      end
+
+      # generate all the contained TwitterXXX objects
+      def each
+        # 
+      end
+
+      # create TwitterUser object from raw info
+      def user
+      end
+      # create Tweet object from raw tweet hash
+      def tweet
+      end
+      # ... and so forth
+    end
+      
+I'll ignore the gory details; view the source if you're interested.
+
+
+h4. Running the script
+
+Here, again, is the code (in full!) for the twitter_request_parser.rb script.
+
+    #
+    # Instantiate each incoming request.
+    # Stream out the contained classes it generates.
+    #
+    class TwitterRequestParser < Wukong::Streamer::StructStreamer
+      def process request
+        request.parse do |obj|
+          yield obj
+        end
+      end
+    end
+
+    # This makes the script go.
+    Wukong::Script.new(TwitterRequestParser, nil).run
+
+That last line is the runner: it makes this a Wukong script with a map phase only.  (We'll add in a reducer later on.)
+