wukong 3.0.0.pre2 → 3.0.0.pre3

data/examples/rake_helper.rb

@@ -4,7 +4,11 @@ require 'gorillib/data_munging'
 require 'pry'
 require 'rake'
 
+<<<<<<< HEAD
+PROJ_ROOT = (ENV['PROJ_CONTENTS'] || File.expand_path('..', File.dirname(__FILE__)))
+=======
 BOOK_ROOT = (ENV['BOOK_CONTENTS'] || File.expand_path('..', File.dirname(__FILE__)))
+>>>>>>> produce_consume
 
 Settings.define :mini, type: :boolean, default: false, description: "use sample data or full data?"
 Settings.resolve!
@@ -14,11 +18,19 @@ Settings[:mini_slug] = Settings.mini ? "-sample" : ""
 task :force
 
 Pathname.register_paths(
+<<<<<<< HEAD
+  proj_root: PROJ_ROOT,
+  root: [:proj_root],
+  code: [:proj_root, 'code'],
+  data: [:proj_root, 'data'],
+  work: [:proj_root, 'tmp'],
+=======
   book_root: BOOK_ROOT,
   root: [:book_root],
   code: [:book_root, 'code'],
   data: [:book_root, 'data'],
   work: [:book_root, 'tmp'],
+>>>>>>> produce_consume
   )
 
 require 'rake/name_space'

data/examples/ruby_project/Gemfile

@@ -0,0 +1,7 @@
+# This is a minimal Gemfile that lets Wukong boot up (but not work).
+gem 'configliere'
+gem 'gorillib'
+gem 'multi_json'
+gem 'vayacondios-client'
+gem 'log4r'
+gem 'eventmachine'

data/examples/ruby_project/README.md

@@ -0,0 +1,6 @@
+This directory contains the skeleton of a Ruby project.  It's used to
+test the behavior of Wukong's command line tools within such a
+context.
+
+The Gemfile here contains the minimal subset of gems that let Wukong
+boot.

data/examples/serverlogs/geo_ip_mapping/munge_geolite.rb

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+require 'wukong'
+require 'gorillib/type/ip_address'
+require 'gorillib/pathname'
+require 'gorillib/model/serialization'
+require 'gorillib/model/positional_fields'
+
+Pathname.register_paths(
+  maxmind: '../../../data/server_logs/maxmind',
+  geolite_ip_blocks: [:maxmind, 'GeoLiteCity_20121002/GeoLiteCity-Blocks.csv'],
+  geolite_locations: [:maxmind, 'GeoLiteCity_20121002/GeoLiteCity-Location.csv'],
+  )
+
+class IpGeo
+  include Gorillib::Model
+  include Gorillib::Model::PositionalFields
+  #
+  field :location_id, Integer
+  field :longitude,   Float  # note: longitude, latitude
+  field :latitude,    Float
+  #
+  field :country_id,  String, blankish: ["", nil, '""']
+  field :admin1_id,   String, blankish: ["", nil, '""']
+  field :city,        String, blankish: ["", nil, '""']
+  #
+  field :postal_code, String, blankish: ["", nil, '""']
+  field :metro_code,  String, blankish: ["", nil, '""']
+  field :area_code,   String, blankish: ["", nil, '""']
+end
+
+lines = 0
+LOCATIONS = {}
+locations_file = Pathname.of(:geolite_locations).open(encoding: "ISO-8859-1")
+locations_file.readline; locations_file.readline
+locations_file.
+  # readlines[0..1000].
+  each do |line|
+  location_id, country_id, admin1_id, city, postal_code, latitude, longitude, metro_code, area_code = line.chomp.gsub(/"/, '').split(',', 9)
+  LOCATIONS[location_id.to_i] = [ location_id.to_i, longitude.to_f, latitude.to_f, country_id, admin1_id, city, postal_code, metro_code, area_code ]
+end
+
+module IpCensus
+  class IpBlocksMapper < Wukong::Streamer::RecordStreamer
+
+    def initialize(*)
+      super
+      @last = IpNumeric.from_dotted('0.116.0.0')
+    end
+
+    def recordize line
+      beg_ip, end_ip, location_id = line.gsub(/\"/, '').split(",", 3)
+      [IpNumeric.new(Integer(beg_ip)), IpNumeric.new(Integer(end_ip)), Integer(location_id)]
+    end
+
+    # Use the regex to break line into fields
+    # Emit each record as flat line
+    def process(beg_ip, end_ip, location_id, &block)
+      emit_range(IpRange.new(@last, beg_ip), 0, &block) if @last != beg_ip.to_int
+      emit_range IpRange.new(beg_ip, end_ip), location_id, &block
+      @last = end_ip.to_i + 1
+    end
+
+    def emit_range(rng, location_id)
+      location = LOCATIONS[location_id]
+      warn "No location #{location_id.inspect}" unless location || (location_id == 0)
+      rng.bitness_blocks(16).each do |blk_min, blk_max|
+        # raise [blk_min, blk_max, blk_min.to_hex[0..-5], blk_max.to_hex[0..-5]].inspect if blk_min.to_hex[0..-3] != blk_max.to_hex[0..-3]
+        yield [
+          blk_min.to_hex[0..3],
+          blk_min.to_hex[4..-1],
+          blk_max.to_hex[4..-1],
+          blk_min.to_i,
+          blk_max.to_i,
+          location,
+        ].flatten
+      end
+    end
+
+  end
+end
+
+Wukong::Script.new(IpCensus::IpBlocksMapper, nil).run

data/examples/serverlogs/models/logline.rb

@@ -0,0 +1,102 @@
+# # Parse logs to TSV
+#
+# bzcat  data/star_wars_kid.log.bz2 | head -n 100200 | tail -n 100 > data/swk-100.log
+# cat data/swk-100.tsv
+# cat data/swk-100.log | ./apache_log_parser.rb --map | wu-lign | cutc 150
+# cat data/swk-100.log | ./apache_log_parser.rb --map > data/swk-100.tsv
+# ./histograms.rb --run data/star_wars_kid.log data/star_wars_kid.tsv
+
+# # Histograms
+#
+# cat data/swk-100.tsv | ./histograms.rb --map | wu-lign
+# cat data/swk-hist-map.tsv | ./histograms.rb --reduce
+# ./histograms.rb --run data/star_wars_kid.tsv data/star_wars_kid-pages_by_hour.tsv
+
+# # Sessionize
+#
+# cat data/swk-100.tsv | ./histograms.rb --map | wu-lign
+# cat data/swk-hist-map.tsv | ./histograms.rb --reduce
+# ./histograms.rb --run data/star_wars_kid.tsv data/star_wars_kid-pages_by_hour.tsv
+
+class Logline
+  include Gorillib::Model
+  include Gorillib::Model::PositionalFields
+
+  field :ip_address,    String
+  field :rfc_1413,      String
+  field :userid,        String
+  #
+  field :requested_at,  Time
+  field :http_method,   String
+  field :path,          String
+  field :protocol,      String
+  field :response_code, Integer
+  field :bytesize,      Integer, blankish: ['', nil, '-']
+  field :referer,       String
+  field :user_agent,    String
+
+  def day_hr
+    [visit_time.year, visit_time.month, visit_time.day, visit_time.hour].join
+  end
+
+  def receive_requested_at(val)
+    if %r{(\d+)/(\w+)/(\d+):(\d+):(\d+):(\d+)\s([\+\-]\d\d)(\d\d)} === val
+      day, mo, yr, hour, min, sec, tz1, tz2 = [$1, $2, $3, $4, $5, $6, $7, $8]
+      val = Time.new(yr.to_i, MONTHS[mo], day.to_i,
+        hour.to_i, min.to_i, sec.to_i, "#{tz1}:#{tz2}")
+    end
+    super(val)
+  end
+
+  # Use the regex to break line into fields
+  # Emit each record as flat line
+  def self.parse(line)
+    mm = LOG_RE.match(line.chomp) or return BadRecord.new('no match', line)
+    new(* mm.captures)
+  end
+
+  def page_type
+    file_ext = path[FILE_EXT_RE]
+    case file_ext
+    when nil                        then 'page'
+    when '.wmv'                     then 'video'
+    when '.html','.shtml'           then 'page'
+    when '.css', '.js'              then 'asset'
+    when '.png', '.gif', '.ico'     then 'image'
+    when '.wmv'                     then 'image'
+    when '.pl','.asp','.jsp','.cgi' then 'page'
+    else                                 'other'
+    end
+  end
+
+  # Matches a file extension
+  FILE_EXT_RE = %r{\.[^/]+\z}
+
+  # Map of abbreviated months to date number.
+  MONTHS = { 'Jan' => 1, 'Feb' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6, 'Jul' => 7, 'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => 11, 'Dec' => 12, }
+
+  #
+  # Regular expression to parse an apache log line.
+  #
+  # 83.240.154.3 - - [07/Jun/2008:20:37:11 +0000] "GET /faq HTTP/1.1" 200 569 "http://infochimps.org/search?query=your_mom" "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
+  #
+  LOG_RE = Regexp.compile(%r{\A
+               (\S+)           # ip_address     83.240.154.3
+             \s(\S+)           # rfc_1413       -
+             \s(\S+)           # userid         -
+
+           \s\[(\d+/\w+/\d+    # date part      [07/Jun/2008
+               :\d+:\d+:\d+    # time part      :20:37:11
+             \s[\+\-]\S*)\]    # timezone       +0000]
+
+        \s\"(?:(\S+)           # http_method    "GET
+             \s(\S+)           # path           /faq
+    \s(HTTP/[\d\.]+)|-)\"      # protocol       HTTP/1.1"
+
+             \s(\d+)           # response_code  200
+             \s(\d+|-)         # bytesize       569
+           \s\"([^\"]*)\"      # referer        "http://infochimps.org/search?query=CAC"
+           \s\"([^\"]*)\"      # user_agent     "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
+          \z}x)
+
+end

data/examples/serverlogs/visit_paths/common.rb

@@ -0,0 +1,4 @@
+$LOAD_PATH.unshift File.expand_path('../../../lib', File.dirname(__FILE__))
+require          'configliere'
+require          'wukong/script'
+require_relative '../models/logline'

data/examples/serverlogs/visit_paths/page_counts.pig

@@ -0,0 +1,48 @@
+
+
+LOAD  common_pages       FROM 'data/common_pages' AS (ip:chararray, from_path:chararray, into_path:chararray);
+
+--
+-- Build adjacency list <A pr B,C,D> from edges (<A B>, <A C>, <A D>)
+-- 
+
+adj_list_j         = GROUP common_pages BY from_path;
+adj_list           = FOREACH adj_list_j GENERATE
+  group                   AS from_path,
+  1.0F                    AS pagerank:float,
+  common_pages.(dest)     AS into_paths
+  ;
+STORE adj_list INTO 'data/pagerank/pr_iter_00';
+
+
+--
+-- Iterate pagerank <A pr_00 B,C,D> to become <A pr_01 B,C,D>
+--
+
+--   find partial share: A.rank / A.into_paths.length
+--   dispatch <into_path partial_share> to each page
+sent_shares  = FOREACH adj_list GENERATE
+        FLATTEN(into_paths)                         AS path,
+        (float)(pagerank / (float)SIZE(into_paths)) AS share:float;
+
+--   dispatch <from_path into_paths>    to yourself, so you have the links still around
+sent_edges   = FOREACH adj_list GENERATE
+        from_path AS path, into_paths;
+
+--   assemble all the received shared, and the self-sent edge list;
+rcvd_shares  = COGROUP sent_edges BY path INNER, sent_shares BY path PARALLEL $PARALLEL;
+
+--   calculate the new rank, and emit a record that looked just like the input.
+next_iter    = FOREACH rcvd_shares {
+        raw_rank    = (float)SUM(sent_shares.share);
+        -- treat the case that a node has no in links                   
+        damped_rank = ((raw_rank IS NOT NULL AND raw_rank > 1.0e-12f) ? raw_rank*0.85f + 0.15f : 0.0f);
+        GENERATE
+                group         AS from_path,
+                damped_rank   AS rank,
+                FLATTEN(sent_edges.into_paths)
+       ; };
+
+STORE next_iter INTO 'data/pagerank/pr_iter_01';
+
+

data/examples/serverlogs/visit_paths/serverlogs-01-parse-script.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+require_relative './common'
+
+class ApacheLogParser < Wukong::Streamer::LineStreamer
+  # create a Logline object from each record and serialize it flat to disk
+  def process line
+    yield Logline.parse(line)
+  end
+end
+
+Wukong.run( ApacheLogParser )

data/examples/serverlogs/visit_paths/serverlogs-02-histograms-full.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+require_relative './common'
+
+# cat data/swk-100.tsv      | ./histograms.rb --map | sort > data/swk-hist-map.tsv
+# cat data/swk-hist-map.tsv | ./histograms.rb --reduce     > data/swk-hist.tsv
+
+class HistogramsMapper < Wukong::Streamer::ModelStreamer
+  self.model_klass = Logline
+  def process visit
+    yield [visit.path, visit.day_hr]
+  end
+end
+
+class HistogramsReducer < Wukong::Streamer::Reducer
+  def get_key(path, day_hr)
+    [path, day_hr]
+  end
+  def start!(*args)
+    @count = 0
+    super
+  end
+  def accumulate(path, day_hr)
+    @count += 1
+  end
+  def finalize
+    yield [key, @count]
+  end
+end
+
+# Wukong.run( HistogramsMapper )
+Wukong.run( HistogramsMapper, HistogramsReducer, :sort_fields => 3 )

data/examples/serverlogs/visit_paths/serverlogs-02-histograms-mapper.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require_relative './common'
+
+class HistogramsMapper < Wukong::Streamer::ModelStreamer
+  self.model_klass = Logline
+  def process visit
+    yield [visit.path, visit.day_hr]
+  end
+end
+
+# Wukong.run( HistogramsMapper )
+Wukong.run( HistogramsMapper )

data/examples/serverlogs/visit_paths/serverlogs-03-breadcrumbs-full.rb

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+require          'configliere'
+Settings.define :page_types, type: Array, default: ['page', 'video'], description: "Acceptable page types"
+require_relative './common'
+
+#
+# Group all visitors, and then troll through all the pages they've visited
+# breaking each into distinct visits (where more than an [hour|day|whatever]
+# separate subsequent pageviews
+#
+
+#
+# Mapper parses log files and created a visitor_id from the visitor's user_id,
+# cookie or ip. It emits
+#
+#    <visitor_id>  <datetime>   <url_path>
+#
+# where the partition key is visitor_id, and we sort by visitor_id and datetime.
+#
+class BreadcrumbsMapper < Wukong::Streamer::ModelStreamer
+  self.model_klass = Logline
+  def process visit, *args
+    # return unless Settings.page_types.include?(visit.page_type)
+    yield [visit.ip, visit.day_hr, visit.requested_at.to_i, visit.path]
+  end
+end
+
+#
+# Reducer:
+#
+# The reducer is given all page requests for the given visitor id, sorted by
+# timestamp.
+#
+# It group by visits (pageviews separated by more than DISTINCT_VISIT_TIMEGAP)
+# and emits
+#
+#     trail        <visitor_id> <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
+#
+# where the last is a comma-separated string of URL encoded paths (any internal comma is converted to %2C).
+#
+# You can instead emit
+#
+#     page_trails  <page1>      <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
+#     page_trails  <page2>      <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
+#     ....
+#     page_trails  <pagen>      <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
+#
+# to discover all trails passing through a given page.
+class BreadcrumbsReducer < Wukong::Streamer::Reducer
+  def get_key ip, day_hr, itime, path, *args
+    [ip]
+  end
+  def start!(*args)
+    @path_times = []
+    super
+  end
+  def accumulate ip, day_hr, itime, path, *args
+    @path_times << "(#{itime},#{path})"
+  end
+  def finalize
+    path_times_str = ("{" << @path_times.join(",") << "}")
+    yield [key, path_times_str]
+  end
+end
+
+
+Wukong.run( BreadcrumbsMapper, BreadcrumbsReducer, :sort_fields => 2 )

data/examples/serverlogs/visit_paths/serverlogs-04-page_page_edges-full.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+require_relative './common'
+
+class BreadcrumbsMapper < Wukong::Streamer::ModelStreamer
+  self.model_klass = Logline
+  def process visit, *args
+    # return unless Settings.page_types.include?(visit.page_type)
+    yield [visit.ip, visit.requested_at.to_i, visit.path]
+  end
+end
+
+class BreadcrumbEdgesReducer < Wukong::Streamer::Reducer
+  def get_key(ip, itime, path)
+    [ip]
+  end
+  def start!(*args)
+    @paths = Set.new
+    super
+  end
+  def accumulate ip, itime, path
+    @paths << path
+  end
+
+  # for each pair of paths, emit the edge in both directions
+  def finalize
+    @paths = @paths.to_a
+    while @paths.present?
+      from = @paths.shift
+      @paths.each do |into|
+        yield [key, from, into]
+        yield [key, into, from]
+      end
+    end
+  end
+end
+
+
+Wukong.run( BreadcrumbsMapper, BreadcrumbEdgesReducer, :sort_fields => 2 )