wukong 3.0.0.pre2 → 3.0.0.pre3

data/examples/munging/wikipedia/n1_subuniverse/n1_nodes.pig

@@ -1,18 +0,0 @@
-/*
- * This script generates the list of all nodes in the 1-neighborhood of the specified node.
- * 
- * Output Format:
- * node_id:int
- */
-
-%default UNDIRECTED_PAGELINKS  '/data/results/wikipedia/full/undirected_pagelinks' -- all edges in the pagelink graph
--- %default HUB1               13692155                                            -- Philosophy
-%default HUB1                  494528786                                           -- Chimpanzee
-%default HUB2                  482846027                                           -- Elephant
-%default N1_NODES_OUT          '/data/results/wikipedia/mini/nodes'                -- where output will be stored
-
-undirected_pagelinks = LOAD     '$UNDIRECTED_PAGELINKS' AS (node_a:int, node_b:int, a_into_b:int, b_into_a:int, is_symmetric:int);
-spokes               = FILTER   undirected_pagelinks BY (node_a == $HUB1) OR (node_b == $HUB1) OR (node_a == $HUB2) OR (node_b == $HUB2);
-neighbors            = FOREACH  spokes GENERATE ((node_a == $HUB) ? node_b : node_a) AS node;
-distinct_neighbors   = DISTINCT neighbors;
-STORE distinct_neighbors INTO '$N1_NODES_OUT';

data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb

@@ -1,21 +0,0 @@
-#!/usr/bin/env ruby
-
-# Sample pig load statement:
-#
-#  page_metadata = LOAD '$page_metadata' AS (id:int, namespace:int, title:chararray, 
-#    restrictions:chararray, counter:long, is_redirect:int, is_new:int, random:float, 
-#    touched:int, page_latest:int, len:int);
-
-require 'wukong'
-require 'wukong/streamer/sql_streamer'
-require 'wukong/streamer/encoding_cleaner'
-
-module PageMetadataExtractor
-  class Mapper < Wukong::Streamer::SQLStreamer
-    include Wukong::Streamer::EncodingCleaner 
-    columns [:int, :int, :string, :string, :int, 
-             :int, :int, :float, :string, :int, :int]
-   end
-end
-
-Wukong::Script.new(PageMetadataExtractor::Mapper, nil).run

data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb.old

@@ -1,27 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'wukong'
-
-load '/home/dlaw/dev/wukong/examples/wikipedia/munging_utils.rb'
-
-module PagesToTSV
-  class Mapper < Wukong::Streamer::LineStreamer
-
-    COLUMNS=  [:int, :int, :string, :string, :int, 
-               :int, :int, :float, :string, :int, :int]
-  
-    def initialize
-      @sql_parser = MungingUtils::SQLParser.new(COLUMNS)
-    end
-
-    def process(line, &blk)
-      @sql_parser.parse(line,&blk)
-    end
-  end
-end
-
-# go to town
-Wukong::Script.new(
-  PagesToTSV::Mapper,
-  nil
-).run

data/examples/munging/wikipedia/pagelinks/augment_pagelinks.pig

@@ -1,29 +0,0 @@
-/*
- A script to generate Wikipedia page graph edge list
- Accepts as input 2 tsvs: list of pages and list of links
- Link table should initially be formatted as from_page_id, into_namespace, into_title
- Assumes that the combination of namespace and title uniquely identifies a page
- 
- Output Format:
- from_id:int, into_id:int, from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray
-*/
-
-%default PAGE_METADATA           '/data/results/wikipedia/full/page_metadata' -- page metadata for all Wikipedia pages
-%default EXTRACTED_PAGELINKS     '/data/scratch/wikipedia/full/pagelinks'     -- raw extracted pagelinks
-%default AUGMENTED_PAGELINKS_OUT '/data/results/wikipedia/full/pagelinks'     -- augmented pagelinks
-
-page_metadata = LOAD '$PAGE_METADATA' AS (id:int, namespace:int, title:chararray, 
-  restrictions:chararray, counter:long, is_redirect:int, is_new:int, 
-  random:float, touched:int, page_latest:int, len:int);
-links = LOAD '$EXTRACTED_PAGELINKS' AS (from_id:int, into_namespace:int, into_title:chararray);
-
-first_pass_j = JOIN page_metadata BY id RIGHT OUTER, links BY from_id;
-first_pass   = FOREACH first_pass_j GENERATE 
-  links::from_id AS from_id, page_metadata::namespace AS from_namespace, page_metadata::title AS from_title,
-  links::into_namespace AS into_namespace, links::into_title AS into_title;
-second_pass_j = JOIN page_metadata BY (namespace, title) RIGHT OUTER, first_pass BY (into_namespace, into_title);
-second_pass   = FOREACH second_pass_j GENERATE 
-  first_pass::from_id, page_metadata::id,
-  first_pass::from_namespace, first_pass::from_title, 
-  first_pass::into_namespace, first_pass::into_title;
-STORE second_pass INTO '$AUGMENTED_PAGELINKS_OUT';

data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'wukong'
-require 'wukong/streamer/sql_streamer'
-require 'wukong/streamer/encoding_cleaner'
-
-module PagelinksExtractor
-  class Mapper < Wukong::Streamer::SQLStreamer
-    include Wukong::Streamer::EncodingCleaner
-    columns [:int, :int, :string]
-  end
-end
-
-Wukong::Script.new(PagelinksExtractor::Mapper, nil).run

data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb.old

@@ -1,25 +0,0 @@
-#!/usr/bin/env ruby
-require 'wukong'
-
-load '/home/dlaw/dev/wukong/examples/wikipedia/munging_utils.rb'
-
-module PagelinksToTSV
-  class Mapper < Wukong::Streamer::LineStreamer
-
-    COLUMNS = [:int, :int, :string]
-
-    def initialize
-      @sql_parser = MungingUtils::SQLParser.new(COLUMNS)
-    end
-
-    def process(line, &blk)
-      @sql_parser.parse(line, &blk)
-    end
-  end
-end
-
-# go to town
-Wukong::Script.new(
-  PagelinksToTSV::Mapper,
-  nil
-).run

data/examples/munging/wikipedia/pagelinks/undirect_pagelinks.pig

@@ -1,29 +0,0 @@
-/*
- * Takes a directed edge list and transforms it into an undirected edge list 
- * that stores edge direction as metadata.
- * 
- * Input table should be of the format (from_id:int, into_id:int ... )
- *
- * Output format:
- * from_id:int, into_id:int, a_into_b:int , b_into_a:int, symmetric:int
- *
- * a_into_b, b_into_a, and symmetric are really booleans.
- */
-
-%default AUGMENTED_PAGELINKS      '/data/results/wikipedia/full/pagelinks'            -- all wikipedia pagelinks (see augment_pagelinks.pig)
-%default UNDIRECTED_PAGELINKS_OUT '/data/results/wikipedia/full/undirected_pagelinks' -- undirected pagelinks
-
-edges = LOAD '$AUGMENTED_PAGELINKS' AS (from:int, into:int);
-edges_sorted = FOREACH edges GENERATE 
-  ((from <= into)? from : into) AS node_a,
-  ((from <= into)? into : from) AS node_b,
-  ((from <= into)? 1 : 0) AS a_to_b,
-  ((from <= into)? 0 : 1) AS b_to_a;
-edges_grouped = GROUP edges_sorted by (node_a, node_b);
-edges_final = FOREACH edges_grouped GENERATE 
-  group.node_a AS node_a,
-  group.node_b AS node_b,
-  ((SUM(edges.$2) > 0) ? 1:0) AS a_into_b,
-  ((SUM(edges.$3) > 0) ? 1:0) AS b_into_a,
-  ((SUM(edges.$2) > 0 AND SUM(edges.$3) > 0) ? 1:0) as symmetric:int;
-STORE edges final INTO '$UNDIRECTED_PAGELINKS_OUT';

data/examples/munging/wikipedia/pageviews/augment_pageviews.pig

@@ -1,32 +0,0 @@
-/*
- * Augments raw pageview data with page ID.
- * Pageview stats are *theoretically* uniquely keyed by namespace 
- * and title, so that is what is used to join pageviews with page_metadata.
- *
- * In practice, the original pageview stats only give the URL visited, and
- * reliably extracting namespace and title from the URL is difficult. Additionally, 
- * page names change, redirects happen, and many other small things can go 
- * wrong with the join. All pageview data is kept in the final table, but 
- * the page id will be blank in rows where the join failed.
- *
- * Output format:
- * page_id:int, namespace:int, title:chararray, num_visitors:long, 
- * date:int, time:int, epoch_time:long, day_of_week:int
- */
-
-%default PAGE_METADATA           '/data/results/wikipedia/full/page_metadata' -- page metadata for all Wikipedia pages
-%default EXTRACTED_PAGEVIEWS     '/data/scratch/wikipedia/full/pageviews'     -- raw extracted pageview stats (see extract_pageviews.rb)
-%default AUGMENTED_PAGEVIEWS_OUT '/data/results/wikipedia/full/pageviews'     -- where output will be stored
-
-page_metadata = LOAD '$PAGE_METADATA' AS 
- (id:int, namespace:int, title:chararray, 
-  restrictions:chararray, counter:long, is_redirect:int, is_new:int, 
-  random:float, touched:int, page_latest:int, len:int);
-pageviews = LOAD '$EXTRACTED_PAGEVIEWS' AS (namespace:int, title:chararray, 
-  num_visitors:long, date:int, time:int, epoch_time:long, day_of_week:int);
-
-first_join = JOIN page_metadata BY (namespace, title) RIGHT OUTER, pageviews BY (namespace, title);
-final = FOREACH first_join GENERATE
-  page_metadata::id, pageviews::namespace, pageviews::title, pageviews::num_visitors,
-  pageviews::date, pageviews::time, pageviews::epoch_time, pageviews::day_of_week;
-STORE final INTO '$AUGMENTED_PAGEVIEWS_OUT';

data/examples/munging/wikipedia/pageviews/extract_pageviews.rb

@@ -1,85 +0,0 @@
-#!/usr/bin/env ruby
-# encoding:UTF-8
-
-# Pig output format:
-# namespace:int, title:chararray, num_visitors:long, date:int, time:int, epoch_time:long, day_of_week:int
-
-$:.unshift '/home/dlaw/dev/wukong_og/lib'
-$:.unshift '/home/dlaw/dev/gorillib/lib'
-
-require 'uri'
-require 'pathname'
-require 'json'
-require 'wukong'
-require 'wukong/streamer'
-require 'wukong/streamer/encoding_cleaner'
-load '/home/dlaw/dev/wukong/examples/munging/wikipedia/utils/munging_utils.rb'
-
-ENV['map_input_file'] ||= 'pagecounts-20071222-100000.gz'
-
-class String
-  def is_enwiki?
-    return (not (self =~ /^en /).nil?)
-  end
-
-  def is_after_enwiki?
-    return (not (self =~ /^(e[o-z][a-z]*|[f-z][a-z]+) /).nil?)
-  end
-end
-
-module PageviewsExtractor
-  class Mapper < Wukong::Streamer::LineStreamer
-    include Wukong::Streamer::EncodingCleaner
-    include MungingUtils
-
-    ns_json_file = File.open("/home/dlaw/dev/wukong/examples/munging/wikipedia/utils/namespaces.json",'r:UTF-8')
-    NAMESPACES = JSON.parse(ns_json_file.read)
-
-    # the filename strings are formatted as
-    # pagecounts-YYYYMMDD-HH0000.gz
-    def time_from_filename(filename)
-      parts = filename.split('-')
-      year = parts[1][0..3].to_i
-      month = parts[1][4..5].to_i
-      day = parts[1][6..7].to_i
-      hour = parts[2][0..1].to_i
-      return Time.new(year,month,day,hour)
-    end
-
-    def process line
-      # we only want enwiki lines
-      return if @done
-      if line.is_after_enwiki?
-        @done = true
-        return
-      end
-      return if not line.is_enwiki?
-      # we have an enwiki line - process it!
-      fields = line.split(' ')[1..-1]
-      out_fields = []
-      # add the namespace
-      namespace = nil
-      if fields[0].include? ':'
-        namespace = NAMESPACES[fields[0].split(':')[0]]
-        out_fields << (namespace || '0')
-      else
-        out_fields << '0'
-      end
-      # add the title
-      if namespace.nil?
-        out_fields << URI.unescape(fields[0])
-      else
-        out_fields << URI.unescape(fields[0][(fields[0].index(':')||-1)+1..-1])
-      end
-      # add number of visitors in the hour
-      out_fields << fields[2]
-      # grab date info from filename
-      file = Pathname.new(ENV['map_input_file']).basename
-      time = time_from_filename(file.to_s)
-      out_fields += time_columns_from_time(time)
-      yield out_fields
-    end
-  end
-end
-
-Wukong::Script.new(PageviewsExtractor::Mapper, Wukong::Streamer::LineStreamer).run

data/examples/munging/wikipedia/pig_style_guide.md

@@ -1,25 +0,0 @@
-# Pig Style Guide
-
-- Everything except names should be in all caps. E.g.
-
-		first_join = JOIN pages BY (namespace,title) 
-			RIGHT OUTER, pageviews BY (namespace, title);
-
-- Group and align columns in the script in ways that make sense. Don't be afraid of newlines. E.g.
-
-		second_pass   = FOREACH second_pass_j GENERATE 
- 		  first_pass::from_id, pages::id,
-		  first_pass::from_namespace, first_pass::from_title, 
-		  first_pass::into_namespace, first_pass::into_title;
-		  
-- Columns that form an important sub-set of the table's data should be easily accessible as a unit. 
-
-	E.g. The edge list above has the from and into ids in the first and second columns, making it easy to just get an edge list of ids without the additional metadata.
-
-- When at all possible, you should include sample LOAD statements in the comments for your script. This makes it easy to use the output of your script
-
-- Parameterize as much as possible. All paths should be parameterized. 
-
-- Parameters should be in all caps, e.g. $NODE.
-
-- Parameters should have defaults if at all possible. When you define the default, also include a comment describing the parameter.

data/examples/munging/wikipedia/redirects/redirects_page_metadata.pig

@@ -1,19 +0,0 @@
-/*
- * Filters the page metadata table, leaving only pages that 
- * are redirects.
- *
- * Output Format (same as page_metadata):
- * (id:int, namespace:int, title:chararray, restrictions:chararray, 
- * counter:long, is_redirect:int, is_new:int, random:float,  touched:int, 
- * page_latest:int, len:int)
- */
-
-%default PAGE_METADATA '/data/results/wikipedia/full/page_metadata' -- page metdata for all pages in Wikipedia
-%default REDIRECTS_OUT '/data/results/wikipedia/full/redirect_page_metadata' -- place to store page metdata for redirects
-
-page_metadata = LOAD '$PAGE_METADATA' AS (id:int, namespace:int, title:chararray, 
-  restrictions:chararray, counter:long, is_redirect:int, is_new:int, random:float, 
-  touched:int, page_latest:int, len:int);
-
-redirects = FILTER page_metadata BY (is_redirect == 1);
-STORE redirects INTO '$REDIRECTS_OUT';

data/examples/munging/wikipedia/subuniverse/sub_articles.pig

@@ -1,23 +0,0 @@
-/*
- * This script filters the articles table, leaving only the articles
- * in the specified subuniverse.
- *
- * Output format:
- * page_id:int, title:chararray, namespace:int, rev_date:int, rev_time:int, 
- * rev_epoch_time:long, rev_dow:int, article_text:chararray
- */
-
-%default ARTICLES         '/data/results/wikipedia/full/articles' -- all articles in the wikipedia corpus
-%default SUB_NODES        '/data/results/wikipedia/mini/nodes'    -- all nodes in the subuniverse
-%default SUB_ARTICLES_OUT '/data/results/wikipedia/mini/articles' -- where output will be stored
-
-articles = LOAD '$ARTICLES' AS (page_id:int, title:chararray, namespace:int, 
-  rev_date:int, rev_time:int, rev_epoch_time:long, rev_dow:int, article_text:chararray);
-sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
-sub_articles_unfiltered = JOIN articles BY id, sub_nodes BY node_id;
-sub_articles = FOREACH sub_articles_unfiltered GENERATE
-  articles::page_id AS page_id, articles::title AS title, articles::namespace AS namespace,
-  articles::rev_date AS rev_date, articles::rev_time AS rev_time,
-  articles::rev_epoch_time AS rev_epoch_time, articles::rev_dow AS rev_dow,
-  articles::article_text AS article_text;
-STORE sub_articles INTO '$SUB_ARTICLES_OUT';

data/examples/munging/wikipedia/subuniverse/sub_page_metadata.pig

@@ -1,24 +0,0 @@
-/*
- * This script filters the page metadata table, leaving only the pages 
- * in the specified subuniverse.
- *
- * Output format (same as page_metadata):
- * id:int, namespace:int, title:chararray, restrictions:chararray, counter:long, 
- * is_redirect:int, is_new:int, random:float, touched:int, page_latest:int, len:int
- */
-
-%default PAGE_METADATA         '/data/results/wikipedia/full/page_metadata' -- metadata for all pages in the wikipedia corpus
-%default SUB_NODES             '/data/results/wikipedia/mini/nodes'         -- all nodes in the subuniverse
-%default SUB_PAGE_METADATA_OUT '/data/results/wikipedia/mini/page_metadata' -- where output will be stored
-
-page_metadata = LOAD '$PAGE_METADATA' AS (id:int, namespace:int, title:chararray, 
-  restrictions:chararray, counter:long, is_redirect:int, is_new:int, random:float, 
-                                          touched:int, page_latest:int, len:int);
-sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
-sub_page_metadata_unfiltered = JOIN page_metadata BY id, sub_nodes BY node_id;
-sub_page_metadata = FOREACH sub_page_metadata_unfiltered GENERATE
-  page_metadata::id, page_metadata::namespace, page_metadata::title, 
-  page_metadata::restrictions, page_metadata::counter, page_metadata::is_redirect, 
-  page_metadata::is_new, page_metadata::random, page_metadata::touched, 
-  page_metadata::page_latest, page_metadata::len;
-STORE sub_page_metadata INTO '$SUB_PAGE_METADATA_OUT';

data/examples/munging/wikipedia/subuniverse/sub_pagelinks_from.pig

@@ -1,22 +0,0 @@
-/*
- * This script filters the pagelinks table, leaving only the pagelinks
- * that start within supplied subuniverse.
- * 
- * Output format (same as augmented_pagelinks):
- * from_id:int, into_id:int, from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray
- */
-
-%default PAGELINKS         '/data/results/wikipedia/full/pagelinks' -- all edges in the pagelink graph (must be *directed*)
-%default SUB_NODES         '/data/results/wikipedia/mini/nodes'     -- all nodes in the subuniverse
-%default SUB_PAGELINKS_OUT '/data/results/wikipedia/mini/pagelinks' -- where output will be stored
-
-all_pagelinks = LOAD '$PAGELINKS' AS (from_id:int, into_id:int, 
-  from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray);
-sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
-
-sub_pagelinks_from = JOIN all_pagelinks BY from_id, sub_nodes BY node_id;
-sub_pagelinks = FOREACH sub_pagelinks_from GENERATE
-  all_pagelinks::from_id, all_pagelinks::into_id, 
-  all_pagelinks::from_namespace, all_pagelinks::from_title,  
-  all_pagelinks::into_namespace, all_pagelinks::into_title;
-STORE sub_pagelinks INTO '$SUB_PAGELINKS_OUT';

data/examples/munging/wikipedia/subuniverse/sub_pagelinks_into.pig

@@ -1,22 +0,0 @@
-/*
- * This script filters the pagelinks table, leaving only the pagelinks
- * that terminate within supplied subuniverse.
- * 
- * Output format (same as augment_pagelinks):
- * node_a:int, node_b:int, a_into_b:int, b_into_a:int, is_symmetric:int
- */
-
-%default PAGELINKS         '/data/results/wikipedia/full/pagelinks' -- all edges in the pagelink graph (must be *directed*)
-%default SUB_NODES         '/data/results/wikipedia/mini/nodes'     -- all nodes in the subuniverse
-%default SUB_PAGELINKS_OUT '/data/results/wikipedia/mini/pagelinks' -- where output will be stored
-
-all_pagelinks = LOAD '$PAGELINKS' AS (from_id:int, into_id:int, 
-  from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray);
-sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
-
-sub_pagelinks_into = JOIN all_pagelinks BY into_id, sub_nodes BY node_id;
-sub_pagelinks = FOREACH sub_pagelinks_into GENERATE
-  all_pagelinks::from_id, all_pagelinks::into_id, 
-  all_pagelinks::from_namespace, all_pagelinks::from_title,  
-  all_pagelinks::into_namespace, all_pagelinks::into_title;
-STORE sub_pagelinks INTO '$SUB_PAGELINKS_OUT';

data/examples/munging/wikipedia/subuniverse/sub_pagelinks_within.pig

@@ -1,26 +0,0 @@
-/*
- * This script filters the pagelinks table, leaving only the pagelinks
- * that start and end within supplied subuniverse.
- * 
- * Output format (same as augment_pagelinks):
- * from_id:int, into_id:int, from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray
- */
-
-%default PAGELINKS         '/data/results/wikipedia/full/undirected_pagelinks' -- all edges in the pagelink graph
-%default SUB_NODES         '/data/results/wikipedia/mini/nodes'                -- all nodes in the subuniverse
-%default SUB_PAGELINKS_OUT '/data/results/wikipedia/mini/pagelinks'            -- where output will be stored
-
-all_pagelinks = LOAD '$PAGELINKS' AS (from_id:int, into_id:int, 
-  from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray);
-sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
-
-sub_pagelinks_in = JOIN all_pagelinks BY from_id, sub_nodes BY node_id;
-sub_pagelinks_unfiltered = JOIN sub_pagelinks_in BY into_id, sub_nodes BY node_id;
-sub_pagelinks = FOREACH sub_pagelinks_unfiltered GENERATE
-  sub_pagelinks_in::all_pagelinks::from_id,
-  sub_pagelinks_in::all_pagelinks::into_id,
-  sub_pagelinks_in::all_pagelinks::from_namespace,
-  sub_pagelinks_in::all_pagelinks::from_title,
-  sub_pagelinks_in::all_pagelinks::into_namespace,
-  sub_pagelinks_in::all_pagelinks::into_title;
-STORE sub_pagelinks INTO '$SUB_PAGELINKS_OUT';