wukong 1.4.6 → 1.4.7

data/CHANGELOG.textile

@@ -1,3 +1,15 @@
+h2. Wukong v1.4.7 2010-03-05
+
+Lots more examples:
+* examples/stats/avg_value_frequency.rb does an Average Value Frequency histogram
+* examples/server_logs has a quite useful apache log file parser
+* Made the base streamer use each_record, opening the door for alternative record injection (eg Datamapper!)
+* wukong/streamer/counting_reducer.rb is an um reducer and it counts things.
+
+h2. Wukong v1.4.6 2010-01-26
+
+* A HELLA AWESOME working example from retail web analytics by @lenbust
+
 h2. Wukong v1.4.5 2010-01-18
 
 * In @--run=local@ mode, you can use '-' alone as a filename to indicate STDIN / STDOUT as input/output respectively.

data/README.textile

@@ -220,6 +220,7 @@ Patches submitted by:
 * ruby interpreter path fix by "Yuichiro MASUI":http://github.com/masuidrive - masui at masuidrive.jp - http://blog.masuidrive.jp/
 
 Thanks to:
+* "Fredrik Möllerstrand (@lenbust)":http://twitter.com/lenbust for the examples/contrib/jeans working example
 * "Brad Heintz":http://www.bradheintz.com/no1thing/talks/ for his early feedback
 * "Phil Ripperger":http://blog.pdatasolutions.com for his "wukong in the Amazon AWS cloud":http://blog.pdatasolutions.com/post/191978092/ruby-on-hadoop-quickstart tutorial.
 

data/examples/apache_log_parser.rb

@@ -1,22 +1,45 @@
 #!/usr/bin/env ruby
 $: << File.dirname(__FILE__)+'/../lib'
+require 'rubygems'
 require 'wukong'
 
+MONTHS = {
+  'Jan' => '01',
+  'Feb' => '02',
+  'Mar' => '03',
+  'Apr' => '04',
+  'May' => '05',
+  'Jun' => '06',
+  'Jul' => '07',
+  'Aug' => '08',
+  'Sep' => '09',
+  'Oct' => '10',
+  'Nov' => '11',
+  'Dec' => '12',
+}
 module ApacheLogParser
   class Mapper < Wukong::Streamer::LineStreamer
 
-    # regular expression for apache-style log lines
-    # note that we strip out the google analytics listener.
-    LOG_RE = %r{\A
-           (\d+\.\d+\.\d+\.\d+)                         # IP addr
-         \s([^\s]+)\s                                   # -
-         \s([^\s]+)                                     # -
-         \s\[(\d\d/\w+/\d+):(\d\d:\d\d:\d\d)([^\]]*)\]  # [07/Jun/2008:20:37:11 +0000]
-         \s(\d+)                                        # 400
-         \s"([^\"]*(?:\" \+ gaJsHost \+ \"[^\"]*)?)"    # "GET /faq" + gaJsHost + "google-analytics.com/ga.js HTTP/1.1"
-         \s(\d+)                                        # 173
-         \s"([^\"]*)" "([^\"]*)" "([^\"]*)"             #  "-" "-" "-"
-      \z}x
+    #
+    # Regular expression to parse an apache log line.
+    #
+    # 83.240.154.3 - - [07/Jun/2008:20:37:11 +0000] "GET /faq HTTP/1.1" 200 569 "http://infochimps.org/search?query=CAC" "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
+    #
+    LOG_RE = Regexp.compile(%r{\A
+           (\S+)                        # ip                  83.240.154.3
+         \s(\S+)                        # j1                  -
+         \s(\S+)                        # j2                  -
+       \s\[(\d+)/(\w+)/(\d+)            # date part           [07/Jun/2008
+          :(\d+):(\d+):(\d+)            # time part           :20:37:11
+         \s(\+.*)\]                     # timezone            +0000]
+    \s\"(?:(\S+)                        # http_method         "GET
+         \s(\S+)                        # path                /faq
+         \s(\S+)|-)"                    # protocol            HTTP/1.1"
+         \s(\d+)                        # response_code       200
+         \s(\d+)                        # duration            569
+       \s\"([^\"]*)\"                   # referer             "http://infochimps.org/search?query=CAC"
+       \s\"([^\"]*)\"                   # ua                  "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
+      \z}x)
 
     # Use the regex to break line into fields
     # Emit each record as flat line
@@ -24,42 +47,28 @@ module ApacheLogParser
       line.chomp
       m = LOG_RE.match(line)
       if m
-        ip, j1, j2, datepart, timepart, tzpart, resp, req, j3, ref, ua, j4 = m.captures
-        req_date = DateTime.parse("#{datepart} #{timepart} #{tzpart}").to_flat
-        req, method, path, protocol = parse_request(req)
-        yield [:logline, method, path, protocol, ip, j1, j2, req_date, resp, req, j3, ref, ua, j4]
+        (ip, j1, j2,
+          ts_day, ts_mo, ts_year,
+          ts_hour, ts_min, ts_sec, req_tz,
+          http_method, path, protocol,
+          response_code, duration,
+          referer, ua, *cruft) = m.captures
+        # DateTime.parse("#{datepart} #{timepart}").to_flat # this takes way too long
+        req_date = [ts_year, MONTHS[ts_mo], ts_day].join("")
+        req_time = [ts_hour, ts_min, ts_sec].join("")
+        yield [:logline, ip, req_date, req_time, http_method, protocol, path, response_code, duration, referer, ua, req_tz]
       else
         yield [:unparseable, line]
       end
     end
 
-
-    def parse_request req
-      m = %r{\A(\w+) (.*) (\w+/[\w\.]+)\z}.match(req)
-      if m
-        [''] + m.captures
-      else
-        [req, '', '', '']
-      end
-    end
-
   end
+end
 
+Wukong::Script.new(ApacheLogParser::Mapper, nil, :sort_fields => 7).run
+
+# 55.55.155.55 - - [04/Feb/2008:11:37:52 +0000] 301 "GET /robots.txt HTTP/1.1" 185 "-" "WebAlta Crawler/2.0 (http://www.webalta.net/ru/about_webmaster.html) (Windows; U; Windows NT 5.1; ru-RU)" "-"
 
-  class Reducer < Wukong::Streamer::LineStreamer
-  end
 
-  # Execute the script
-  class Script < Wukong::Script
-    def reduce_command
-      "/usr/bin/uniq"
-    end
-    def default_options
-      super.merge :sort_fields => 8 # , :reduce_tasks => 0
-    end
-  end
 
-  Script.new(Mapper,nil).run
-end
 
-# 55.55.155.55 - - [04/Feb/2008:11:37:52 +0000] 301 "GET /robots.txt HTTP/1.1" 185 "-" "WebAlta Crawler/2.0 (http://www.webalta.net/ru/about_webmaster.html) (Windows; U; Windows NT 5.1; ru-RU)" "-"

data/examples/server_logs/breadcrumbs.rb

@@ -0,0 +1,36 @@
+#
+# Group all visitors, and then troll through all the pages they've visited
+# breaking each into distinct visits (where more than an [hour|day|whatever]
+# separate subsequent pageviews
+#
+
+#
+# Mapper parses log files and created a visitor_id from the visitor's user_id,
+# cookie or ip. It emits
+#
+#    <visitor_id>  <datetime>   <url_path>
+#
+# where the partition key is visitor_id, and we sort by visitor_id and datetime.
+#
+
+#
+# Reducer:
+#
+# The reducer is given all page requests for the given visitor id, sorted by
+# timestamp.
+#
+# It group by visits (pageviews separated by more than DISTINCT_VISIT_TIMEGAP)
+# and emits
+#
+#     trail        <visitor_id> <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
+#
+# where the last is a comma-separated string of URL encoded paths (any internal comma is converted to %2C).
+#
+# You can instead emit
+#
+#     page_trails  <page1>      <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
+#     page_trails  <page2>      <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
+#     ....
+#     page_trails  <pagen>      <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
+#
+# to discover all trails passing through a given page.

data/examples/server_logs/user_agent.rb

@@ -0,0 +1,40 @@
+
+
+# For later, if we want to parse user agents:
+#   http://code.google.com/p/browserscope/source/browse/trunk/models/user_agent.py
+#   http://www.useragentstring.com/pages/All/
+#   http://github.com/jaxn/parse-user-agent
+#   http://code.google.com/p/browserscope/wiki/UserAgentParsing
+#   http://code.google.com/p/ua-parser/source/browse/
+#   http://github.com/shenoudab/active_device/tree/master/lib/active_device/
+
+
+#
+# * Mozilla based
+# * Mozilla version
+# * X11 based
+# * Security
+# * OS
+# * CPU family
+# * Language Tag
+# * Renderer (i.e. Webkit, Trident, Presto)
+# * Renderer Version
+# * I don't see a utility for the "KHTML" and "like Gecko" bits, but whatever.
+# * Based on
+# * Browser Build (not really sure about this either)
+
+# * Browser Family (i.e. Firefox, IE, Chrome, etc..)
+# * Project Name (optional, i.e. Namoroka, Shiretoko)
+# * Major Version
+# * Minor Version
+# * Version Third Bit
+# * Version Fourth Bit
+# * Open Question: How should we handle the "alpha/beta" bit, like apre1? I'm inclined to say we put it in its own datapoint and let people group together how ever they want, but not leave it attached to any of the version bits.
+
+# Bot
+# Brand
+# Browser
+# Engine
+# Handset
+# Model
+# OS

data/examples/stats/avg_value_frequency.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+# run like so:
+# $> ruby average_value_frequecy.rb --run=local data/stats.tsv data/avf_out.tsv
+require 'rubygems'
+require 'wukong'
+
+#
+# Calculate the average value frequency (AVF) for each data row. AVF for a data
+# point with m attributes is defined as:
+#
+#     avf = (1/m)* sum (frequencies of attributes 1..m)
+#
+# so with the data
+#
+#      1       15      30      25
+#      2       10      10      20
+#      3       50      30      30
+#
+# for the first row, avf = (1/3)*(1+2+1) ~= 1.33. An outlier is identified by
+# a low AVF.
+#
+module AverageValueFrequency
+  # Names for each column's attribute, in order
+  ATTR_NAMES = %w[length width height]
+
+  class HistogramMapper < Wukong::Streamer::RecordStreamer
+    # unroll each row from
+    #     [id,   val1,   val2, ....]
+    # into
+    #     [attr1,   val1]
+    #     [attr2,   val2]
+    #     ...
+    def process id, *values
+      ATTR_NAMES.zip(values).each do |attr, val|
+        yield [attr, val]
+      end
+    end
+  end
+
+  #
+  # Build a histogram of values
+  #
+  class HistogramReducer < Wukong::Streamer::CountingReducer
+    # use the attr and val as the key
+    def get_key attr, val=nil, *_
+      [attr, val]
+    end
+  end
+
+  class AvfRecordMapper < Wukong::Streamer::RecordStreamer
+    # average the frequency of each value
+    def process id, *values
+      sum = 0.0
+      ATTR_NAMES.zip(values).each do |attr, val|
+        sum += histogram[ [attr, val] ].to_i
+      end
+      avf = sum / ATTR_NAMES.length.to_f
+      yield [id, avf, *values]
+    end
+
+    # Load the histogram from a tab-separated file with
+    #   attr    val   freq
+    def histogram
+      return @histogram if @histogram
+      @histogram = { }
+      File.open(options[:histogram_file]).each do |line|
+        attr, val, freq = line.chomp.split("\t")
+        @histogram[ [attr, val] ] = freq
+      end
+      @histogram
+    end
+  end
+end
+
+Settings.use :commandline, :define
+Settings.define :histogram,      :description => "Run the first pass to calculate a histogram"
+Settings.define :avf,            :description => "Run the second pass, to run back over the records with the histogram and find the AVF for each row."
+Settings.define :histogram_file, :description => "File to load the histogram from (supply name of the  output file from first pass)"
+Settings.resolve!
+if Settings[:histogram]
+  Wukong::Script.new(AverageValueFrequency::HistogramMapper, AverageValueFrequency::HistogramReducer).run
+elsif Settings[:avf]
+  Wukong::Script.new(AverageValueFrequency::AvfRecordMapper, nil).run
+else
+  raise "Please specify either --histogram (for first round) or --avf (second round)"
+end

data/examples/stats/data/avg_value_frequency.tsv

@@ -0,0 +1,3 @@
+1	15	30	25
+2	10	10	20
+3	50	30	30

data/lib/wukong/script.rb

@@ -121,6 +121,7 @@ module Wukong
     def initialize mapper_klass, reducer_klass, extra_options={}
       self.options = Settings.dup
       options.resolve!
+      options.merge! self.default_options
       options.merge! extra_options
       self.mapper_klass  = mapper_klass
       self.reducer_klass = reducer_klass

data/lib/wukong/script/hadoop_command.rb

@@ -27,6 +27,8 @@ module Wukong
     Settings.define :timeout,                :jobconf => true, :description => 'mapred.task.timeout', :wukong => true
     Settings.define :reuse_jvms,             :jobconf => true, :description => 'mapred.job.reuse.jvm.num.tasks', :wukong => true
     Settings.define :respect_exit_status,    :jobconf => true, :description => 'stream.non.zero.exit.is.failure', :wukong => true
+    Settings.define :noempty,                                  :description => "don't create zero-byte reduce files (hadoop mode only)", :wukong => true
+    # mapred.linerecordreader.maxlength :description => "Safeguards against corrupted data: lines longer than this (in bytes) are treated as bad records."
 
     # emit a -jobconf hadoop option if the simplified command line arg is present
     # if not, the resulting nil will be elided later
@@ -66,7 +68,8 @@ module Wukong
     end
 
     def hadoop_other_args
-      extra_str_args = [ options[:extra_args] ]
+      extra_str_args  = [ options[:extra_args] ]
+      extra_str_args               += ' -lazyOutput' if options[:noempty]  # don't create reduce file if no records
       options[:reuse_jvms]          = '-1'     if (options[:reuse_jvms] == true)
       options[:respect_exit_status] = 'false'  if (options[:ignore_exit_status] == true)
       extra_hsh_args = [:map_speculative, :timeout, :reuse_jvms, :respect_exit_status].map{|opt| jobconf(opt)  }

data/lib/wukong/streamer.rb

@@ -11,5 +11,6 @@ module Wukong
     autoload :AccumulatingReducer,  'wukong/streamer/accumulating_reducer'
     autoload :ListReducer,          'wukong/streamer/list_reducer'
     autoload :UniqByLastReducer,    'wukong/streamer/uniq_by_last_reducer'
+    autoload :CountingReducer,      'wukong/streamer/counting_reducer'
   end
 end

data/lib/wukong/streamer/base.rb

@@ -19,9 +19,8 @@ module Wukong
       def stream
         Log.info("Streaming on:\t%s" % [Script.input_file]) unless Script.input_file.blank?
         before_stream
-        $stdin.each do |line|
-          record = recordize(line.chomp)
-          next unless record
+        each_record do |line|
+          record = recordize(line.chomp) or next
           process(*record) do |output_record|
             emit output_record
           end
@@ -29,6 +28,10 @@ module Wukong
         after_stream
       end
 
+      def each_record &block
+        $stdin.each(&block)
+      end
+
       # Called exactly once, before streaming begins
       def before_stream
       end

data/lib/wukong/streamer/counting_reducer.rb

@@ -0,0 +1,25 @@
+module Wukong
+  module Streamer
+
+    #
+    # Count the number of records for each key.
+    #
+    class CountingReducer < AccumulatingReducer
+      attr_accessor :count
+
+      # start the sum with 0 for each key
+      def start! *_
+        self.count = 0
+      end
+      # ... and count the number of records for this key
+      def accumulate *_
+        self.count += 1
+      end
+      # emit [key, count]
+      def finalize
+        yield [key, count].flatten
+      end
+    end
+
+  end
+end

data/wukong.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{wukong}
-  s.version = "1.4.6"
+  s.version = "1.4.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2010-01-26}
+  s.date = %q{2010-03-04}
   s.description = %q{  Treat your dataset like a:
 
       * stream of lines when it’s efficient to process by lines
@@ -109,7 +109,11 @@ Gem::Specification.new do |s|
      "examples/rank_and_bin.rb",
      "examples/run_all.sh",
      "examples/sample_records.rb",
+     "examples/server_logs/breadcrumbs.rb",
+     "examples/server_logs/user_agent.rb",
      "examples/size.rb",
+     "examples/stats/avg_value_frequency.rb",
+     "examples/stats/data/avg_value_frequency.tsv",
      "examples/word_count.rb",
      "lib/wukong.rb",
      "lib/wukong/bad_record.rb",
@@ -145,6 +149,7 @@ Gem::Specification.new do |s|
      "lib/wukong/streamer/base.rb",
      "lib/wukong/streamer/count_keys.rb",
      "lib/wukong/streamer/count_lines.rb",
+     "lib/wukong/streamer/counting_reducer.rb",
      "lib/wukong/streamer/filter.rb",
      "lib/wukong/streamer/line_streamer.rb",
      "lib/wukong/streamer/list_reducer.rb",
@@ -170,7 +175,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://mrflip.github.com/wukong}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.5}
+  s.rubygems_version = %q{1.3.6}
   s.summary = %q{Wukong makes Hadoop so easy a chimpanzee can use it.}
   s.test_files = [
     "spec/spec_helper.rb",
@@ -193,7 +198,10 @@ Gem::Specification.new do |s|
      "examples/pagerank/pagerank_initialize.rb",
      "examples/rank_and_bin.rb",
      "examples/sample_records.rb",
+     "examples/server_logs/breadcrumbs.rb",
+     "examples/server_logs/user_agent.rb",
      "examples/size.rb",
+     "examples/stats/avg_value_frequency.rb",
      "examples/word_count.rb"
   ]
 

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: wukong
 version: !ruby/object:Gem::Version 
-  version: 1.4.6
+  prerelease: false
+  segments: 
+  - 1
+  - 4
+  - 7
+  version: 1.4.7
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -9,39 +14,45 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-26 00:00:00 -06:00
+date: 2010-03-04 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: addressable
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: extlib
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: htmlentities
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id003
 description: "  Treat your dataset like a:\n\n      * stream of lines when it\xE2\x80\x99s efficient to process by lines\n      * stream of field arrays when it\xE2\x80\x99s efficient to deal directly with fields\n      * stream of lightweight objects when it\xE2\x80\x99s efficient to deal with objects\n\n  Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.\n"
 email: flip@infochimps.org
 executables: 
@@ -141,7 +152,11 @@ files:
 - examples/rank_and_bin.rb
 - examples/run_all.sh
 - examples/sample_records.rb
+- examples/server_logs/breadcrumbs.rb
+- examples/server_logs/user_agent.rb
 - examples/size.rb
+- examples/stats/avg_value_frequency.rb
+- examples/stats/data/avg_value_frequency.tsv
 - examples/word_count.rb
 - lib/wukong.rb
 - lib/wukong/bad_record.rb
@@ -177,6 +192,7 @@ files:
 - lib/wukong/streamer/base.rb
 - lib/wukong/streamer/count_keys.rb
 - lib/wukong/streamer/count_lines.rb
+- lib/wukong/streamer/counting_reducer.rb
 - lib/wukong/streamer/filter.rb
 - lib/wukong/streamer/line_streamer.rb
 - lib/wukong/streamer/list_reducer.rb
@@ -211,18 +227,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Wukong makes Hadoop so easy a chimpanzee can use it.
@@ -247,5 +265,8 @@ test_files:
 - examples/pagerank/pagerank_initialize.rb
 - examples/rank_and_bin.rb
 - examples/sample_records.rb
+- examples/server_logs/breadcrumbs.rb
+- examples/server_logs/user_agent.rb
 - examples/size.rb
+- examples/stats/avg_value_frequency.rb
 - examples/word_count.rb