RubyGems - wukong - Versions diffs - 0.1.1 - Mend

wukong 0.1.1

Files changed (143) hide show

data/LICENSE.textile +107 -0
data/README.textile +166 -0
data/bin/cutc +30 -0
data/bin/cuttab +5 -0
data/bin/greptrue +8 -0
data/bin/hdp-cat +3 -0
data/bin/hdp-catd +3 -0
data/bin/hdp-du +81 -0
data/bin/hdp-get +3 -0
data/bin/hdp-kill +3 -0
data/bin/hdp-ls +10 -0
data/bin/hdp-mkdir +3 -0
data/bin/hdp-mv +3 -0
data/bin/hdp-parts_to_keys.rb +77 -0
data/bin/hdp-ps +3 -0
data/bin/hdp-put +3 -0
data/bin/hdp-rm +11 -0
data/bin/hdp-sort +29 -0
data/bin/hdp-stream +29 -0
data/bin/hdp-stream-flat +18 -0
data/bin/hdp-sync +17 -0
data/bin/hdp-wc +67 -0
data/bin/md5sort +20 -0
data/bin/tabchar +5 -0
data/bin/uniqc +3 -0
data/bin/wu-hist +3 -0
data/bin/wu-lign +177 -0
data/bin/wu-sum +30 -0
data/doc/INSTALL.textile +41 -0
data/doc/LICENSE.textile +107 -0
data/doc/README-tutorial.textile +163 -0
data/doc/README-wulign.textile +59 -0
data/doc/README-wutils.textile +128 -0
data/doc/TODO.textile +61 -0
data/doc/UsingWukong-part1-setup.textile +2 -0
data/doc/UsingWukong-part2-scraping.textile +2 -0
data/doc/UsingWukong-part3-parsing.textile +132 -0
data/doc/code/api_response_example.txt +20 -0
data/doc/code/parser_skeleton.rb +38 -0
data/doc/hadoop-nfs.textile +51 -0
data/doc/hadoop-setup.textile +29 -0
data/doc/index.textile +124 -0
data/doc/intro_to_map_reduce/MapReduceDiagram.graffle +0 -0
data/doc/links.textile +42 -0
data/doc/overview.textile +91 -0
data/doc/pig/PigLatinExpressionsList.txt +122 -0
data/doc/pig/PigLatinReferenceManual.html +19134 -0
data/doc/pig/PigLatinReferenceManual.txt +1640 -0
data/doc/tips.textile +116 -0
data/doc/usage.textile +102 -0
data/doc/utils.textile +48 -0
data/examples/README.txt +17 -0
data/examples/and_pig/sample_queries.rb +128 -0
data/examples/apache_log_parser.rb +53 -0
data/examples/count_keys.rb +56 -0
data/examples/count_keys_at_mapper.rb +57 -0
data/examples/graph/adjacency_list.rb +74 -0
data/examples/graph/breadth_first_search.rb +79 -0
data/examples/graph/gen_2paths.rb +68 -0
data/examples/graph/gen_multi_edge.rb +103 -0
data/examples/graph/gen_symmetric_links.rb +53 -0
data/examples/package-local.rb +100 -0
data/examples/package.rb +96 -0
data/examples/pagerank/README.textile +6 -0
data/examples/pagerank/gen_initial_pagerank_graph.pig +57 -0
data/examples/pagerank/pagerank.rb +88 -0
data/examples/pagerank/pagerank_initialize.rb +46 -0
data/examples/pagerank/run_pagerank.sh +19 -0
data/examples/rank_and_bin.rb +173 -0
data/examples/run_all.sh +47 -0
data/examples/sample_records.rb +44 -0
data/examples/size.rb +60 -0
data/examples/word_count.rb +95 -0
data/lib/wukong.rb +11 -0
data/lib/wukong/and_pig.rb +62 -0
data/lib/wukong/and_pig/README.textile +12 -0
data/lib/wukong/and_pig/as.rb +37 -0
data/lib/wukong/and_pig/data_types.rb +30 -0
data/lib/wukong/and_pig/functions.rb +50 -0
data/lib/wukong/and_pig/generate.rb +85 -0
data/lib/wukong/and_pig/generate/variable_inflections.rb +82 -0
data/lib/wukong/and_pig/junk.rb +51 -0
data/lib/wukong/and_pig/operators.rb +8 -0
data/lib/wukong/and_pig/operators/compound.rb +29 -0
data/lib/wukong/and_pig/operators/evaluators.rb +7 -0
data/lib/wukong/and_pig/operators/execution.rb +15 -0
data/lib/wukong/and_pig/operators/file_methods.rb +29 -0
data/lib/wukong/and_pig/operators/foreach.rb +98 -0
data/lib/wukong/and_pig/operators/groupies.rb +212 -0
data/lib/wukong/and_pig/operators/load_store.rb +65 -0
data/lib/wukong/and_pig/operators/meta.rb +42 -0
data/lib/wukong/and_pig/operators/relational.rb +129 -0
data/lib/wukong/and_pig/pig_struct.rb +48 -0
data/lib/wukong/and_pig/pig_var.rb +95 -0
data/lib/wukong/and_pig/symbol.rb +29 -0
data/lib/wukong/and_pig/utils.rb +0 -0
data/lib/wukong/bad_record.rb +18 -0
data/lib/wukong/boot.rb +47 -0
data/lib/wukong/datatypes.rb +24 -0
data/lib/wukong/datatypes/enum.rb +123 -0
data/lib/wukong/dfs.rb +80 -0
data/lib/wukong/encoding.rb +111 -0
data/lib/wukong/extensions.rb +15 -0
data/lib/wukong/extensions/array.rb +18 -0
data/lib/wukong/extensions/blank.rb +93 -0
data/lib/wukong/extensions/class.rb +189 -0
data/lib/wukong/extensions/date_time.rb +24 -0
data/lib/wukong/extensions/emittable.rb +82 -0
data/lib/wukong/extensions/hash.rb +120 -0
data/lib/wukong/extensions/hash_like.rb +119 -0
data/lib/wukong/extensions/hashlike_class.rb +47 -0
data/lib/wukong/extensions/module.rb +2 -0
data/lib/wukong/extensions/pathname.rb +27 -0
data/lib/wukong/extensions/string.rb +65 -0
data/lib/wukong/extensions/struct.rb +17 -0
data/lib/wukong/extensions/symbol.rb +11 -0
data/lib/wukong/logger.rb +53 -0
data/lib/wukong/models/graph.rb +27 -0
data/lib/wukong/rdf.rb +104 -0
data/lib/wukong/schema.rb +37 -0
data/lib/wukong/script.rb +265 -0
data/lib/wukong/script/hadoop_command.rb +111 -0
data/lib/wukong/script/local_command.rb +14 -0
data/lib/wukong/streamer.rb +13 -0
data/lib/wukong/streamer/accumulating_reducer.rb +89 -0
data/lib/wukong/streamer/base.rb +76 -0
data/lib/wukong/streamer/count_keys.rb +30 -0
data/lib/wukong/streamer/count_lines.rb +26 -0
data/lib/wukong/streamer/filter.rb +20 -0
data/lib/wukong/streamer/line_streamer.rb +12 -0
data/lib/wukong/streamer/list_reducer.rb +20 -0
data/lib/wukong/streamer/preprocess_with_pipe_streamer.rb +22 -0
data/lib/wukong/streamer/rank_and_bin_reducer.rb +145 -0
data/lib/wukong/streamer/set_reducer.rb +14 -0
data/lib/wukong/streamer/struct_streamer.rb +48 -0
data/lib/wukong/streamer/summing_reducer.rb +29 -0
data/lib/wukong/streamer/uniq_by_last_reducer.rb +44 -0
data/lib/wukong/typed_struct.rb +12 -0
data/lib/wukong/wukong_class.rb +21 -0
data/spec/bin/hdp-wc_spec.rb +4 -0
data/spec/spec_helper.rb +0 -0
data/wukong.gemspec +179 -0
metadata +214 -0

data/doc/tips.textile ADDED Viewed

@@ -0,0 +1,116 @@
+h3. Don't Drop ACID while exploring Big Data
+The traditional "ACID quartet":http://en.wikipedia.org/wiki/ACID for relational databases can be re-interpreted in a Big Data context:
+* A -- Associative
+* C -- Commutative
+* I -- Idempotent
+* D -- Distributed
+* (*) -- (and where possible, left in sort order)
+Finally, where possible leave things in sort order by some appropriate index. Clearly I'm not talking about introducing extra unnecessary sorts on ephemeral data.  For things that will be read (and experimented with) much more often than they're written, though, it's worth running a final sort.  Now you can
+* Efficiently index into a massive dataset with binary search
+* Do a direct merge sort on two files with the same sort order
+* Run a reducer directly across the data
+* Assign a synthetic key by just serially numbering lines (either distribute a unique prefix to each mapper
+Note: for files that will live on the DFS, you should usually *not* do a total sort,
+h3. If it's not broken, it's wrong
+Something that goes wrong one in a five million times will crop up hundreds of times in a billion-record collection.
+h3. Error is not normally distributed
+What's more, errors introduced will not in general be normally distributed and their impact may not decrease with increasing data size.
+h3. Encode once, and carefully.
+Encoding violates idempotence.  Data brought in from elsewhere *must* be considered unparsable, ill-formatted and rife with illegal characters.
+* Immediately fix a copy of the original data with as minimal encoding as possible.
+* Follow this with a separate parse stage to emit perfectly well-formed, tab-separated / newline delimited data
+* In this parse stage, encode the data to 7-bits, free of internal tabs, backslashes, carriage return/line feed or control characters. You want your encoding scheme to be
+** perfectly reversible
+** widely implemented
+** easily parseable
+** recognizable: incoming data that is mostly inoffensive (a json record, or each line of a document such as this one) should be minimally altered from its original.  This lets you do rough exploration with sort/cut/grep and friends.
+** !! Involve **NO QUOTING**, only escaping.  I can write a simple regexp to decode entities such as %10, \n or &#10;. This regexp will behave harmlessly with ill-formed data (eg %%10 or &&; or \ at end of line) and is robust against data being split or interpolated.  Schemes such as "quoting: it's bad", %Q{quoting: "just say no"} or <notextile><notextile>tagged markup</notextile></notextile> require a recursive parser. An extra or missing quote mark is almost impossible to backtrack.  And av
+In the absence of some lightweight, mostly-transparent, ASCII-compatible *AND* idempotent encoding scheme lurking in a back closet of some algorithms book -- how to handle the initial lousy payload coming off the wire?
+* For data that is *mostly* text in a western language, you'll do well wiht XML encoding (with <notextile>[\n\r\t\\]</notextile> forced to encode as entities)
+* URL encoding isn't as recognizable, but is also safe. Use this for things like URIs and filenames, or if you want to be /really/ paranoid about escaping.
+* For binary data, Binhex is efficient enough and every toolkit can handle it.  There are more data-efficient ascii-compatible encoding schemes but it's not worth the hassle for the 10% or whatever gain in size.
+* If your payload itself is XML data, consider using \0 (nul) between records, with a fixed number of tab-separated metadata fields leading the XML data, which can then include tabs, newlines, or whatever the hell it wants. No changes are made to the data apart from a quick gsub to remove any (highly illegal) \0 in the XML data itself. A later parse round will convert it to structured hadoop-able data.  Ex:
+{% highlight html %}
+feed_request	20090809101112  200     OK      <?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html lang='en' xml:lang='en' xmlns='http://www.w3.org/1999/xhtml'>
+  <head>
+      <title>infochimps.org &mdash; Find Any Dataset in the World</title>
+{% endhighlight %}
+p. Many of the command line utilities (@cat@, @grep@, etc.) will accept nul-delimited files.
+You may be tempted to use XML around your XML so you can XML while you XML, but this is ultimately only done right by parsing or scrubbing the inputm and at that point you should just translate directly to a reasonable tab/newline format. (Even if that format is tsv-compatible JSON).
+h3. Do your exception handling in-band
+A large, heavily-used cluster will want to have ganglia or "scribe":http://www.cloudera.com/blog/2008/11/02/configuring-and-using-scribe-for-hadoop-log-collection/ or the like collecting and managing log data.  "Splunk":http://www.splunk.com/ is a compelling option I haven't myself used, but it is "broadly endorsed.":http://www.igvita.com/2008/10/22/distributed-logging-syslog-ng-splunk/
+However, it's worth considering another extremely efficient, simple and powerful distributed system for routing massive quantities of data in a structured way, namely wukong|hadoop itself.
+Wukong gives you a BadRecord class -- just rescue errors, pass the full or partial contents of the offending input. and emit the BadRecord instance in-band.  They'll be serialized out along with the rest, and at your preference can be made to reduce to a single instance. Do analysis on them at your leisure; by default, any StructStreamer will silently discard *inbound* BadRecords -- they won't survive past the current generation.
+h3. Don't be afraid to use the command line as an IDE
+%{ highlight sh %}
+  cat /data/foo.tsv | ruby -ne 'puts $_.chomp.scan(/text="([^"]+)"/).join("\t")'
+{% endhighlight %}
+h3. Keys
+* Artificial key: assigned externally, key is not a function of the object's intrinsic values. A social security number is an artificial key. Artificial
+* Natural Key: minimal subset of fields with _intrinsic semantic value_ that _uniquely identify_ the record. My name isn't unique, but my fingerprint is both uniqe and intrinsic.  Given the object (me) you can generate the key, and given the key there's exactly one object (me) that matches.
+h4. other fields
+* Mutable:
+** A user's 'bio' section.
+* Immutable:
+** A user's created_at date is immutable: it doesn't help identify the person but it will never change.
+h4. Natural keys are right for big data
+Synthetic keys suck. They demand locality or a central keymaster.
+* Use the natural key
+* Hash the natural key. This has some drawbacks
+OK, fine. you need a synthetic key
+* Do a total sort, and use nl
+* Generate
+* Use a single reducer to reduce locality. YUCK.
+* have each mapper generate a unique prefix; number each line as "prefix#{line_number}" or whatever.
+How do you get a unique prefix?
+* Distribute a unique prefix to each mapper out-of-band. People using Streaming are out of luck.
+* Use a UUID -- that's what they're for. Drawback: ridiculously long
+* Hash the machine name, PID and timestamp to something short. Check after the
+  fact that uniqueness was achieved. Use the birthday party formula to find out
+  how often this will happen. (In practice, almost never.)
+Working with records that change over time,

data/doc/usage.textile ADDED Viewed

@@ -0,0 +1,102 @@
+---
+layout: default
+title:  Usage notes
+---
+h1(gemheader). {{ site.gemname }} %(small):: usage%
+<notextile><div class="toggle"></notextile>
+h2. How to run a Wukong script
+To run your script using local files and no connection to a hadoop cluster,
+pre. your/script.rb --run=local path/to/input_files path/to/output_dir
+To run the command across a Hadoop cluster,
+pre. your/script.rb --run=hadoop path/to/input_files path/to/output_dir
+You can set the default in the config/wukong-site.yaml file, and then just use @--run@ instead of @--run=something@ --it will just use the default run mode.
+If you're running @--run=hadoop@, all file paths are HDFS paths. If you're running @--run=local@, all file paths are local paths.  (your/script path, of course, lives on the local filesystem).
+You can supply arbitrary command line arguments (they wind up as key-value pairs in the options path your mapper and reducer receive), and you can use the hadoop syntax to specify more than one input file:
+pre. ./path/to/your/script.rb --any_specific_options --options=can_have_vals \
+   --run "input_dir/part_*,input_file2.tsv,etc.tsv" path/to/output_dir
+Note that all @--options@ must precede (in any order) all non-options.
+<notextile></div><div class="toggle"></notextile>
+h2. How to test your scripts
+To run mapper on its own:
+pre. cat ./local/test/input.tsv | ./examples/word_count.rb --map | more
+or if your test data lies on the HDFS,
+pre. hdp-cat test/input.tsv | ./examples/word_count.rb --map | more
+Next graduate to running @--run=local@ mode so you can inspect the reducer.
+<notextile></div><div class="toggle"></notextile>
+h2. What tools does Wukong work with?
+Wukong is friends with "Hadoop":http://hadoop.apache.org/core the elephant, "Pig":http://hadoop.apache.org/pig/ the query language, and the @cat@ on your command line.  We're looking forward to being friends with "martinis":http://datamapper.org and "express trains":http://wiki.rubyonrails.org/rails/pages/ActiveRecord down the road.
+<notextile></div><div class="toggle"></notextile>
+h2. Design
+...
+<notextile></div><div class="toggle"></notextile>
+h2. Caveats
+...
+<notextile></div><div class="toggle"></notextile>
+h2. TODOs
+...
+<notextile></div><div class="toggle"></notextile>
+h2. Note on Patches/Pull Requests
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a future version unintentionally.
+* Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but  bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+<notextile></div><div class="toggle"></notextile>
+h2. Endnotes
+h3. Why is it called Wukong?
+Hadoop, as you may know, is "named after a stuffed elephant.":http://en.wikipedia.org/wiki/Hadoop Since Wukong was started by the "infochimps":http://infochimps.org team, we needed a simian analog.  A Monkey King who journeyed to the land of the Elephant seems to fit the bill:
+bq. Sun Wukong (孙悟空), known in the West as the Monkey King, is the main character in the classical Chinese epic novel Journey to the West. In the novel, he accompanies the monk Xuanzang on the journey to retrieve Buddhist sutras from India.
+bq. Sun Wukong possesses incredible strength, being able to lift his 13,500 jīn (8,100 kg) Ruyi Jingu Bang with ease. He also has superb speed, traveling 108,000 li (54,000 kilometers) in one somersault. Sun knows 72 transformations, which allows him to transform into various animals and objects; he is, however, shown with slight problems transforming into other people, since he is unable to complete the transformation of his tail. He is a skilled fighter, capable of holding his own against the best generals of heaven. Each of his hairs possesses magical properties, and is capable of transforming into a clone of the Monkey King himself, or various weapons, animals, and other objects. He also knows various spells in order to command wind, part water, conjure protective circles against demons, freeze humans, demons, and gods alike. -- ["Sun Wukong's Wikipedia entry":http://en.wikipedia.org/wiki/Wukong]
+The "Jaime Hewlett / Damon Albarn short":http://news.bbc.co.uk/sport1/hi/olympics/monkey that the BBC made for their 2008 Olympics coverage gives the general idea.
+* What's up with Wukong::AndPig?
+** @Wukong::AndPig@ is a small library to more easily generate code for the "Pig":http://hadoop.apache.org/pig data analysis language.  See its "README":wukong/and_pig/README.textile for more.
+<notextile></div></notextile>

data/doc/utils.textile ADDED Viewed

@@ -0,0 +1,48 @@
+<something to tab and align table>
+* uniq                  - report or filter out repeated lines in a file
+**  -c                    produces line<tab>count
+**  --ignore f1,f2,...    discards given fields from consideration. field syntax same as for cut, etc.
+* sort                  - sort lines of text files
+** columns indexed as tab-separated
+** can specify any column order, uses same field spec as cut
+* tsort                 - topological sort of a directed graph
+* cut                   - select portions of each line of a file
+** can reorder columns
+* nl                    - line numbering filter
+** takes prefix, suffix
+** count \t line -OR- line \t count
+* wc                    - word, line, character, and byte count
+** field count (tab-separated fields)
+* paste                 - merge corresponding or subsequent lines of files
+* expand, unexpand      - expand tabs to spaces, and vice versa
+* seq
+* simple row, column sums
+* join                  - relational database operator
+* tac
+* cat                   - concatenate and print files
+* head                  - display first lines of a file
+* tail                  - display the last part of a file
+* shuf
+* split                 - split a file into pieces
+* csplit                - split files based on context
+* tee                   - pipe fitting
+* ls                    - list directory contents.
+* df                    - display free disk space
+* du                    - display disk usage statistics
+** tab-delimited, space aligned
+* od                    - octal, decimal, hex, ASCII dump
+* printf                - formatted output
+* cksum, sum            - display file checksums and block counts
+* md5sum
+* diff
+* comm

data/examples/README.txt ADDED Viewed

@@ -0,0 +1,17 @@
+Examples:
+* sample_records -- extract a random sample from a collection of data
+* word_count
+* apache_log_parser -- example for parsing standard apache webserver log files.
+* wordchains -- solving a word puzzle using breadth-first search of a graph
+* graph -- some generic graph
+* pagerank -- use the pagerank algorithm to find the most 'interesting'
+  (central) nodes of a network graph

data/examples/and_pig/sample_queries.rb ADDED Viewed

@@ -0,0 +1,128 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__) + '/../../lib'
+require 'wukong'         ; include Wukong
+require 'wukong/and_pig' ; include Wukong::AndPig
+# PIG_DIR = '/usr/local/share/pig'
+PIG_DIR = '/public/share/pig'
+# full pathname to the pig executable
+# Wukong::AndPig::PIG_EXECUTABLE = "#{PIG_DIR}/bin/pig"
+Wukong::AndPig::PIG_EXECUTABLE = "/public/bin/pig -x local"
+#
+HDFS_BASE_DIR = 'foo/meta/lang'
+Wukong::AndPig::PigVar.working_dir = HDFS_BASE_DIR
+Wukong::AndPig.comments    = false
+# Wukong::AndPig.emit_dest   = :captured
+Wukong::AndPig::PigVar.emit "REGISTER #{PIG_DIR}/contrib/piggybank/java/piggybank.jar"
+#
+# Load basic types
+#
+# class Token < Struct.new(:rsrc, :context, :user_id, :token, :usages)
+# end
+# :tokens_users_0 << Token.pig_load('meta/datanerds/token_count/users_tokens')
+# :tokens_users_0 << Token.pig_load('/tmp/users_tokens.tsv')
+# :tokens_users   << :tokens_users_0.generate(:user_id, :token, :usages)
+# :tokens_users.checkpoint!
+class Token < TypedStruct.new(
+      [:user_id, Integer], [:token, String], [:usages, Integer])
+end
+:tokens_users << Token.pig_load('/tmp/users_tokens.tsv')
+:tokens_users.describe
+pig_comment %Q{
+# ***************************************************************************
+#
+# Global totals
+#
+# Each row in Tokens lists a (user, token, usages)
+# We want
+#   Sum of all usage counts = total tokens seen in tweet stream.
+#   Number of distinct tokens
+#   Number of distinct users <- different than total in twitter_users.tsv
+#                               because we want only users that say stuff.
+}
+def count_distinct relation, field, options={}
+  result_name = options[:as] || "#{relation.name}_#{field}_count".to_sym
+  a = relation.
+    generate(field).set!.describe.
+    distinct(options).set!
+  result_name << a.
+    group(:all).set!.
+    generate(["COUNT(#{a.relation}.#{field})", :u_count, Integer]).set!
+end
+pig_comment "Count Users"
+tok_users_count = count_distinct(:tokens_users, :user_id).checkpoint!
+pig_comment "Count Tokens"
+tok_tokens_count = count_distinct(:tokens_users, :token, :parallel => 10).checkpoint!
+pig_comment %Q{
+# ***************************************************************************
+#
+# Statistics for each user
+}
+def user_stats users_tokens
+  users_tokens.describe.
+    group(   :user_id).set!.describe.
+    generate(
+      [:group, :user_id],
+      ["(int)COUNT(#{users_tokens.relation})",          :tot_tokens, Integer],
+      [  "(int)SUM(#{users_tokens.relation}.usages)",   :tot_usages, Integer],
+      [   "FLATTEN(#{users_tokens.relation}.token",     :token,      String ],
+      [   "FLATTEN(#{users_tokens.relation}.usages",    :usages,     Integer]).set!.describe.
+    # [   "FLATTEN(#{users_tokens.relation}.(token, usages) )", [:token, :usages], TypedStruct.new([:token, String], [:usages, Integer])]).set!.
+    generate(:user_id, :token, :usages,
+         ["(float)(1.0*usages / tot_usages)", :usage_pct, Float],
+         ["(float)(1.0*usages / tot_usages) * (1.0*(float)usages / tot_usages)", :usage_pct_sq, Float]).set!
+end
+:user_stats << user_stats(:tokens_users)
+:user_stats.describe.checkpoint!
+puts "UserStats               = LOAD    'foo/meta/lang/user_stats' AS (user_id, token, usages, usage_pct, usage_pct_sq) ;"
+UserStats = TypedStruct.new([:user_id, Integer],
+  [:token, String],
+  [:usages, Integer],
+  [:usage_pct, Float],
+  [:usage_pct_sq, Float])
+:user_stats << UserStats.pig_load('foo/meta/lang/user_stats')
+def range_and_dispersion user_stats
+  n_users  = 436
+  n_tokens = 61630
+  token_stats = user_stats.group(:token).set!
+  token_stats = token_stats.foreach(
+      ["(float)SUM(#{user_stats.relation}.usage_pct)    / #{n_users.to_f}",   :avg_uspct ],
+    ["(float)SUM(#{user_stats.relation}.usage_pct_sq)",                     :sum_uspct_sq],
+    ["org.apache.pig.piggybank.evaluation.math.SQRT(
+                        (sum_uspct_sq /436) -
+                        ( (SUM(#{user_stats.relation}.usage_pct)/436.0) * (SUM(#{user_stats.relation}.usage_pct)/436.0) )
+                        )",     :stdev_uspct],
+    ["1 - ( ( stdev_uspct / avg_uspct ) / org.apache.pig.piggybank.evaluation.math.SQRT(436.0 - 1.0) )",  :dispersion],
+    [
+      [:group,                                                                :token,     String     ],
+      ["(int)COUNT(#{user_stats.relation}) ",                                 :range,     Integer     ],
+      ["(int)COUNT(#{user_stats.relation})              / #{n_users.to_f}",   :pct_range,  Integer ],
+      ["(int)SUM(  #{user_stats.relation}.usages)",                           :tot_usages, Integer],
+      ["(float)( 1.0e6*SUM(#{user_stats.relation}.usages) / #{n_tokens.to_f})", :ppm_usages, Float],
+      [:avg_uspct,   :avg_uspct],
+      [:stdev_uspct, :stdev_uspct],
+      [:dispersion,  :dispersion]
+    ]
+    ).set!
+end
+range_and_dispersion(:user_stats).checkpoint!
+Wukong::AndPig.finish

data/examples/apache_log_parser.rb ADDED Viewed

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong'
+module ApacheLogParser
+  class Mapper < Wukong::Streamer::LineStreamer
+    def parse_request req
+      m = %r{\A(\w+) (.*) (\w+/[\w\.]+)\z}.match(req)
+      if m
+        [''] + m.captures
+      else
+        [req, '', '', '']
+      end
+    end
+    # regular expression to match on apache-style log lines
+    #             IP addr              -         -        [07/Jun/2008:20:37:11 +0000]               400   "GET /faq" + gaJsHost + "google-analytics.com/ga.js HTTP/1.1" 173 "-" "-" "-"
+    LOG_RE = %r{\A(\d+\.\d+\.\d+\.\d+) ([^\s]+) ([^\s]+) \[(\d\d/\w+/\d+):(\d\d:\d\d:\d\d)([^\]]*)\] (\d+) "([^\"]*(?:\" \+ gaJsHost \+ \"[^\"]*)?)" (\d+) "([^\"]*)" "([^\"]*)" "([^\"]*)"\z}
+    def process line
+      line.chomp
+      m = LOG_RE.match(line)
+      if m
+        ip, j1, j2, datepart, timepart, tzpart, resp, req, j3, ref, ua, j4 = m.captures
+        req_date = DateTime.parse("#{datepart} #{timepart} #{tzpart}").to_flat
+        req, method, path, protocol = parse_request(req)
+        yield [:logline, method, path, protocol, ip, j1, j2, req_date, resp, req, j3, ref, ua, j4]
+      else
+        yield [:unparseable, line]
+      end
+    end
+  end
+  class Reducer < Wukong::Streamer::LineStreamer
+  end
+  # Execute the script
+  class Script < Wukong::Script
+    def reduce_command
+      "/usr/bin/uniq"
+    end
+    def default_options
+      super.merge :sort_fields => 8 # , :reduce_tasks => 0
+    end
+  end
+  Script.new(Mapper,nil).run
+end
+# 55.55.155.55 - - [04/Feb/2008:11:37:52 +0000] 301 "GET /robots.txt HTTP/1.1" 185 "-" "WebAlta Crawler/2.0 (http://www.webalta.net/ru/about_webmaster.html) (Windows; U; Windows NT 5.1; ru-RU)" "-"