wukong 3.0.0.pre → 3.0.0.pre2

data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+require_relative './dbpedia_common'
+
+# ## Usage
+#
+# Flattens the wikipedia 'enwiki-latest-pages-articles.xml.gz' into a
+# one-line-per-record heap.
+#
+#    examples/munging/wikipedia/dbpedia/extract_geolocations.rb --rm --run \
+#      /data/origin/wikipedia/dbpedia/geo_coordinates_en.nq \
+#      /data/results/wikipedia/dbpedia-geo_coordinates_en.tsv
+#
+
+# ## Sample Pig Schema
+#
+# geolocations = LOAD '$geolocations' AS
+#   (id:long, namespace:int, title:chararray, longitude:float, latitude:float);
+#
+
+module Dbpedia
+  class GeocoordinatesExtractor < Wukong::Streamer::LineStreamer
+    include MungingUtils
+
+    KNOWN_LINE_RE = %r{\A(?:
+         \#\sstarted
+      |  <http://dbpedia\.org/resource/[^>]+>\s
+         <http://(
+            www\.georss\.org/georss/point
+         |  www\.w3\.org/1999/02/22-rdf-syntax-ns\#type
+         |  www\.w3\.org/2003/01/geo/wgs84_pos\#(?:lat|long)
+      )>
+    )}x
+
+    # it's on one line in the actual dataset; split here for readability
+    GEO_RSS_RE = %r{\A
+         <http://dbpedia\.org/resource/(?<title>[^>]+)>
+      \s <http://www\.georss\.org/georss/point>
+      \s "(?<lat>#{DECIMAL_NUM_RE})\s(?<lng>#{DECIMAL_NUM_RE})"@en
+      \s <http://en\.wikipedia\.org/wiki/(?:[^\?]+)\?oldid=(?<article_id>\d+)>
+      \s \.
+    \z}x
+
+    def warn_record(desc, record=nil)
+      record_info = MultiJson.encode(record)[0..1000] rescue "(unencodeable record) #{record.inspect[0..100]}"
+      Log.warn [desc, record_info].join("\t")
+    end
+
+    ARTICLE_NAMESPACE = 0
+
+    # The file is in that godawful semantic web format. Let's fix that.
+    #
+    #     <http://dbpedia.org/resource/Alabama> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.opengis.net/gml/_Feature> <http://en.wikipedia.org/wiki/Alabama?oldid=495507959> .
+    #     <http://dbpedia.org/resource/Alabama> <http://www.w3.org/2003/01/geo/wgs84_pos#lat> "33.0"^^<http://www.w3.org/2001/XMLSchema#float> <http://en.wikipedia.org/wiki/Alabama?oldid=495507959> .
+    #     <http://dbpedia.org/resource/Alabama> <http://www.w3.org/2003/01/geo/wgs84_pos#long> "-86.66666666666667"^^<http://www.w3.org/2001/XMLSchema#float> <http://en.wikipedia.org/wiki/Alabama?oldid=495507959> .
+    #     <http://dbpedia.org/resource/Alabama> <http://www.georss.org/georss/point> "33.0 -86.66666666666667"@en <http://en.wikipedia.org/wiki/Alabama?oldid=495507959> .
+    #
+    # The lines seem to be redundant, with the georss one containing what we need, so just filter for those
+    #
+    def process(line)
+      if not KNOWN_LINE_RE.match(line) then warn_record("Unrecognized line type", line) ; return ; end
+      return unless $1 == 'www.georss.org/georss/point'
+      geo_info = GEO_RSS_RE.match(line)
+      if not geo_info then warn_record("Unrecognized georss line", line) ; return ; end
+
+      result = [
+        geo_info[:article_id],
+        ARTICLE_NAMESPACE, # the dbpedia stuff is all NS 0
+        geo_info[:title],
+        geo_info[:lng],
+        geo_info[:lat],
+      ]
+      yield result
+    end
+  end
+
+end
+
+Wukong::Script.new(Dbpedia::GeocoordinatesExtractor, nil).run

data/examples/munging/wikipedia/dbpedia/extract_links.rb

@@ -0,0 +1,193 @@
+#!/usr/bin/env ruby
+require_relative './dbpedia_common'
+require 'ap'
+
+module Re
+  ##
+  # Container for the character classes specified in
+  # <a href="http://www.ietf.org/rfc/rfc3986.txt">RFC 3986</a>.
+  # Borrowed from the addressable gem
+  module Uri
+    ALPHA      = "a-zA-Z"
+    DIGIT      = "0-9"
+    GEN_DELIMS = "\\:\\/\\?\\#\\[\\]\\@"
+    SUB_DELIMS = "\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\="
+    RESERVED   = GEN_DELIMS + SUB_DELIMS
+    UNRESERVED = ALPHA + DIGIT + "\\-\\.\\_\\~"
+    PCHAR      = UNRESERVED + SUB_DELIMS + "\\:\\@"
+    SCHEME     = ALPHA + DIGIT + "\\-\\+\\."
+    AUTHORITY  = PCHAR
+    PATH       = PCHAR + "\\/"
+    QUERY      = PCHAR + "\\/\\?"
+    FRAGMENT   = PCHAR + "\\/\\?"
+    #
+    PATHSEG    = ""
+  end
+end
+#
+#
+#
+module Dbpedia
+
+  DBLQ_STRING_C   = '"(?<%s>\\\"|[^\"]+)+"'
+  DECIMAL_NUM_RE  = '[\-\+\d]+\.\d+'
+
+  RDF_RES = {
+    dbpedia_ontb:     'http://dbpedia\.org/ontology',
+    dbpedia_ont:      'http://dbpedia\.org/ontology/(?<%s>[\w\/]+)',
+    dbpedia_prop:     'http://dbpedia\.org/property/(?<%s>\w+)',
+    dbpedia_class:    'http://dbpedia\.org/class/(?<%s>[^>\s]+)',
+    dbpedia_rsrc:     "http://dbpedia\\.org/resource/(?<%s>[#{Re::Uri::PCHAR}%%\/]+)",
+    wikipedia_rsrc:   "http://\\w\\w\\.wikipedia\\.org/wiki/(?<%s>[#{Re::Uri::PCHAR}%%\/]+)",
+    dbpedia_value:    '"(?<%s>(?:\\\"|[^\"]+)*)"\\^\\^<http://dbpedia\.org/datatype/(?<%s>[a-zA-Z]+)>',
+    foaf_homepage:    'http://xmlns\.com/foaf/0\.1/homepage',
+    foaf_name:        'http://xmlns\.com/foaf/0\.1/name',
+    foaf_topic:       'http://xmlns\.com/foaf/0\.1/(?:isPrimaryTopicOf|primaryTopic)',
+    foaf_prop:        'http://xmlns\.com/foaf/0\.1/(?<property>\w+)',
+    geonames_rsrc:    'http://sws\.geonames\.org/(?<%s>\d+)/',
+    georss_latlng:    '\"(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\\s(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\"@en',
+    georss_type:      'http://www\.georss\.org/georss/point',
+    musicbrainz_rsrc: 'http://zitgist\.com/music/(?<%s>\w+)/(?<%s>[a-f0-9\-]+)',
+    nytimes_rsrc:     'http://data\.nytimes\.com/(?<%s>[A-Z0-9]+)',
+    purl_subject:     'http://purl\.org/dc/terms/subject',
+    purl_desc:        'http://purl\.org/dc/elements/1\.1/(?<%s>description)',
+    purl_lang:        'http://purl\.org/dc/elements/1\.1/language',
+    rdf_comment:      'http://www\.w3\.org/2000/01/rdf-schema\\#comment',
+    rdf_eol:          '\\.',
+    rdf_float:        '\"(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\"\^\^<http://www\.w3\.org/2001/XMLSchema\#float>',
+    rdf_integer:      '\"(?<%s>\d+)\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#integer>',
+    rdf_date:         '\"(?<%s>\d\d\d\d-\d\d-\d\d)\"\^\^<http://www\.w3\.org/2001/XMLSchema\\#date>',
+    rdf_string:       '\"(?<%s>(?:\\\"|[^\"]+)*)"@(?<%s>\w+)\b',
+    rdf_label:        'http://www\.w3\.org/2000/01/rdf-schema\\#label',
+    rdf_type:         'http://www\.w3\.org/1999/02/22-rdf-syntax-ns\\#type',
+    same_as:          'http://www\.w3\.org/2002/07/owl\\#sameAs',
+    skos_concept:     'http://www\.w3\.org/2004/02/skos/core\\#(?<%s>[a-zA-Z]+)',
+    skos_subject:     'http://www\.w3\.org/2004/02/skos/core\\#subject',
+    wgs_latorlng:     'http://www\.w3\.org/2003/01/geo/wgs84_pos\\#(?:lat|long)',
+    url_loose:        '(?<%s>(?:https?|ftp)://(?:[a-zA-Z0-9\-]+\.)+(?:[a-zA-Z\-]+)[^\s>]*)',
+    wiki_category:    'http://en\.wikipedia\.org/wiki/Category:Futurama?oldid=485425712\\#absolute-line=1',
+    wiki_link_id:     'http://en\.wikipedia\.org/wiki/(?<%s>[^\?]+)\?oldid=(?<%s>\d+)\\#?(?:absolute-line=(?<%s>\d+))?',
+    wiki_link_id_sec: 'http://en\.wikipedia\.org/wiki/(?<%s>[^\?]+)\?oldid=(?<%s>\d+)\\#?(?:section=(?<%s>.*?)\&relative-line=(?<%s>\d+))?(?:&?absolute-line=(?<%s>\d+))?',
+    wordnet_inst:     'http://www\.w3\.org/2006/03/wn/wn20/instances/synset-(?<%s>\w+)-noun-(?<%s>[0-9]+)',
+    yago_class:       'http://dbpedia\.org/class/yago',
+    rdf_value:        '\"(?<%s>
+      -?\d\d\d\d-\d\d-\d\d|-?\d\d\d\d-\d\d|--\d\d-\d\d|
+      [\+\-]?\d+  |
+      [\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?|
+      true|false)\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>integer|date|gYearMonth|gMonthDay|gYear|positiveInteger|nonNegativeInteger|float|double|boolean)>',
+
+    schema_type:      'http://(?<%s>www\\.w3\\.org/2002/07/owl|schema\\.org|dbpedia\\.org/ontology|purl\\.org/ontology|xmlns.com/foaf/0\\.1)[/\#]([^>]+)'
+  }
+
+  idx = 0;
+  RDF_RES.each{|flavor, re_str| Regexp.new(re_str.gsub('%s'){|x| "l#{idx+=1}" }) }
+
+  private
+  def self.r(regexp_name, *args)
+    RDF_RES[regexp_name] % args
+  end
+  public
+
+  MAPPINGS = {
+
+    geo_coordinates:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:georss_type)}>                        \s#{r(:georss_latlng, :lat, :lng)}               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    geo_coord_skip_a:      %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<http://www\.opengis\.net/gml/_Feature>        \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    geo_coord_skip_b:      %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:wgs_latorlng)}>                       \s#{r(:rdf_float, :val)}                         \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    wordnet:               %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_prop, :property)}>            \s<#{r(:wordnet_inst, :wn_class, :wn_idx)}>      \s#{r(:rdf_eol)}  \z}x,
+    geonames:              %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:geonames_rsrc, :geonames_id)}>           \s#{r(:rdf_eol)}  \z}x,
+
+    pnd:                   %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/individualisedPnd>     \s#{r(:rdf_string, :name, :nlang)}               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    disambiguations:       %r{\A<#{r(:dbpedia_rsrc, :generic_wikipedia_id)}> \s<#{r(:dbpedia_ontb)}/wikiPageDisambiguates> \s<#{r(:dbpedia_rsrc, :specific_wikipedia_id)}>  \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    page_ids:              %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/wikiPageID>            \s#{r(:rdf_integer,   :wikipedia_pageid)}        \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}>     \s#{r(:rdf_eol)}  \z}x,
+    redirects:             %r{\A<#{r(:dbpedia_rsrc, :dupe_of)}>              \s<#{r(:dbpedia_ontb)}/wikiPageRedirects>     \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>           \s#{r(:rdf_eol)}   \z}x,
+
+    article_categories:    %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:purl_subject)}>                       \s<#{r(:dbpedia_rsrc, :specific_wikipedia_id)}>  \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    categories_skos:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:skos_concept, :skos_relation)}>          \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    categories_skos_label: %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:skos_concept, :skos_relation)}>       \s#{r(:rdf_string, :val, :val_lang)}             \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    categories_skos_reln:  %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:skos_concept, :skos_relation)}>       \s<#{r(:dbpedia_rsrc, :category_b)}>             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    category_labels:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_label)}>                          \s#{r(:rdf_string,:category_wikipedia_id, :ctl)} \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+
+    abstracts_short:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_comment)}>                        \s#{r(:rdf_string,:abstract, :al)}               \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    abstracts_long:        %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/abstract>              \s#{r(:rdf_string,:abstract, :al)}               \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}xm,
+    titles:                %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_label)}>                          \s#{r(:rdf_string,:wikipedia_id2,  :tl )}        \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    nytimes:               %r{\A<#{r(:nytimes_rsrc, :nyt_id)}>               \s<#{r(:same_as)}>                            \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>           \s#{r(:rdf_eol)}   \z}x,
+
+    external_links:        %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/wikiPageExternalLink>  \s<#{r(:url_loose, :weblink_url)}>               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    homepages:             %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:foaf_homepage)}>                      \s<#{r(:url_loose, :weblink_url)}>               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_links:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:foaf_topic)}>                         \s<#{r(:url_loose, :weblink_url)}>               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_lang:        %r{\A<#{r(:wikipedia_rsrc, :wikipedia_id)}>       \s<#{r(:purl_lang)}>                          \s#{r(:rdf_string, :lang, :wll_lang)}            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_rev:         %r{\A<#{r(:wikipedia_rsrc, :wikipedia_id)}>       \s<#{r(:foaf_topic)}>                         \s<#{r(:dbpedia_rsrc, :wikipedia_id2)}>          \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    musicbrainz:           %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:musicbrainz_rsrc, :musicbrainz_type, :musicbrainz_id)}>    \s#{r(:rdf_eol)}  \z}x,
+
+    properties_map_val:    %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_value,  :val, :val_type)}             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    properties_map_str:    %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_string, :val, :val_lang)}             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    properties_map_foaf:   %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:foaf_prop, :property)}>               \s#{r(:rdf_string, :val, :val_lang)}             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    properties_specmap:    %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:dbpedia_value, :val, :dbpedia_units)}     \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+
+    yago:                  %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:schema_type, :schema, :schema_type)}>    \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+
+    persondata_type:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<http://xmlns.com/foaf/0\.1/Person>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    persondata_reln:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s<#{r(:dbpedia_rsrc, :target_wikipedia_id)}>    \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    persondata_prop:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_value, :val, :val_type)}              \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    persondata_foaf:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:foaf_prop, :property)}>               \s#{r(:rdf_string,:name, :name_lang)}            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+    persondata_desc:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:purl_desc, :property)}>               \s#{r(:rdf_string,:name, :name_lang)}            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
+
+
+    #topical_concepts:      %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>       \s<#{r(:skos_subject)}>                       \s<#{r(:x, )} \z}x,  #<http://dbpedia.org/resource/Futurama> \s<#{r(:wiki_category)}> .
+    # uscensus:               %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>        \s<#{r(:same_as)}>                            \s<#{r(:url_loose, :census_url)}>   \s#{r(:rdf_eol)}  \z}x,
+    # instance_types:        %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:dbpedia_ont, :specific_wikipedia_id)}> }x,#        \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+  }
+
+  # ap MAPPINGS
+
+  class RdfExtractor < Wukong::Streamer::LineStreamer
+    include MungingUtils
+
+    def emit(hsh)
+      super [MultiJson.encode(hsh)]
+    end
+
+    def process(line)
+      return if line =~ /\A(?:\#|$)/
+      if line =~ /=> \w+\.\w+ <=/ then yield [line] ; return ; end
+      MAPPINGS.each do |flavor, re|
+        next unless mm = re.match(line)
+        yield( { flavor: flavor }.merge(mm.as_hash) )
+        return
+      end
+      puts [line]
+    end
+  end
+end
+
+
+# META = {
+#   geo_coordinates:    [:field,      'geo_coordinates_en.nq',                      ],
+#   wordnet:            [:joinkey,    'wordnet_links.nt',                           ],
+#   geonames:           [:joinkey,    'geonames_links.nt',                          ],
+#   properties_specmap: [:properties, 'specific_mappingbased_properties_en.nq',     ],
+#   properties_mapped:  [:properties, 'mappingbased_properties_unredirected_en.nq', ],
+#   pnd:                [:joinkey,    'pnd_en.nq',                                  ],
+#   disambiguations:    [:pagelink,   'disambiguations_unredirected_en.nq',         ],
+#   external_links:     [:weblink,    'external_links_en.nq',                       ],
+#   page_ids:           [:field,      'page_ids_en.nq',                             ],
+#   redirects:          [:pagelink,   'redirects_transitive_en.nt',                 ],
+#   article_categories: [:categories, 'article_categories_en.nq',                   ],
+#   instance_types:     [:categories, 'instance_types_en.nq',                       ],
+#   categories_skos:    [:meta,       'skos_categories_en.nq',                      ],
+#   abstracts_long:     [:field,      'long_abstracts_en.nq',                       ],
+#   abstracts_short:    [:field,      'short_abstracts_en.nq',                      ],
+#   category_labels:    [:meta,       'category_labels_en.nq',                      ],
+#   titles:             [:field,      'labels_en.nq',                               ],
+#   musicbrainz:        [:joinkey,    'musicbrainz_links.nt',                       ],
+#   nytimes:            [:joinkey,    'nytimes_links.nt',                           ],
+#   uscensus:           [:joinkey,    'uscensus_links.nt',                          ],
+#   topical_concepts:   [             'topical_concepts_unredirected_en.nq',        ],
+#   homepages:          [:weblink,    'homepages_en.nq',                            ],
+#   wikipedia_links:    [:field,      'wikipedia_links_en.nq',                      ],
+#   persondata:         [:properties, 'persondata_unredirected_en.nq',              ],
+#   yago:               [:joinkey,    'yago_links.nt',                              ],
+# }
+
+
+Wukong::Script.new(Dbpedia::RdfExtractor, nil).run

data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb

@@ -0,0 +1,20 @@
+module Dbpedia
+
+  class SameasExtractor < Wukong::Streamer::LineStreamer
+    include MungingUtils
+
+    SAME_AS_RE = %r{\A
+         <http://dbpedia\.org/resource/(?<title>[^>]+)>
+      \s <http://www\.w3\.org/2002/07/owl\#sameAs>
+      \s <http://(?<target>[^>]+)>
+      \s \.
+    \z}x
+
+    def recordize(line)
+      same_as = SAME_AS_RE.match(line)
+      if not same_as then warn_record("Unrecognized line type", line) ; return end
+      [same_as[:title], same_as[:target]]
+    end
+  end
+
+end

data/examples/munging/wikipedia/n1_subuniverse/n1_nodes.pig

@@ -0,0 +1,18 @@
+/*
+ * This script generates the list of all nodes in the 1-neighborhood of the specified node.
+ * 
+ * Output Format:
+ * node_id:int
+ */
+
+%default UNDIRECTED_PAGELINKS  '/data/results/wikipedia/full/undirected_pagelinks' -- all edges in the pagelink graph
+-- %default HUB1               13692155                                            -- Philosophy
+%default HUB1                  494528786                                           -- Chimpanzee
+%default HUB2                  482846027                                           -- Elephant
+%default N1_NODES_OUT          '/data/results/wikipedia/mini/nodes'                -- where output will be stored
+
+undirected_pagelinks = LOAD     '$UNDIRECTED_PAGELINKS' AS (node_a:int, node_b:int, a_into_b:int, b_into_a:int, is_symmetric:int);
+spokes               = FILTER   undirected_pagelinks BY (node_a == $HUB1) OR (node_b == $HUB1) OR (node_a == $HUB2) OR (node_b == $HUB2);
+neighbors            = FOREACH  spokes GENERATE ((node_a == $HUB) ? node_b : node_a) AS node;
+distinct_neighbors   = DISTINCT neighbors;
+STORE distinct_neighbors INTO '$N1_NODES_OUT';

data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+# Sample pig load statement:
+#
+#  page_metadata = LOAD '$page_metadata' AS (id:int, namespace:int, title:chararray, 
+#    restrictions:chararray, counter:long, is_redirect:int, is_new:int, random:float, 
+#    touched:int, page_latest:int, len:int);
+
+require 'wukong'
+require 'wukong/streamer/sql_streamer'
+require 'wukong/streamer/encoding_cleaner'
+
+module PageMetadataExtractor
+  class Mapper < Wukong::Streamer::SQLStreamer
+    include Wukong::Streamer::EncodingCleaner 
+    columns [:int, :int, :string, :string, :int, 
+             :int, :int, :float, :string, :int, :int]
+   end
+end
+
+Wukong::Script.new(PageMetadataExtractor::Mapper, nil).run

data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb.old

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'wukong'
+
+load '/home/dlaw/dev/wukong/examples/wikipedia/munging_utils.rb'
+
+module PagesToTSV
+  class Mapper < Wukong::Streamer::LineStreamer
+
+    COLUMNS=  [:int, :int, :string, :string, :int, 
+               :int, :int, :float, :string, :int, :int]
+  
+    def initialize
+      @sql_parser = MungingUtils::SQLParser.new(COLUMNS)
+    end
+
+    def process(line, &blk)
+      @sql_parser.parse(line,&blk)
+    end
+  end
+end
+
+# go to town
+Wukong::Script.new(
+  PagesToTSV::Mapper,
+  nil
+).run

data/examples/munging/wikipedia/pagelinks/augment_pagelinks.pig

@@ -0,0 +1,29 @@
+/*
+ A script to generate Wikipedia page graph edge list
+ Accepts as input 2 tsvs: list of pages and list of links
+ Link table should initially be formatted as from_page_id, into_namespace, into_title
+ Assumes that the combination of namespace and title uniquely identifies a page
+ 
+ Output Format:
+ from_id:int, into_id:int, from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray
+*/
+
+%default PAGE_METADATA           '/data/results/wikipedia/full/page_metadata' -- page metadata for all Wikipedia pages
+%default EXTRACTED_PAGELINKS     '/data/scratch/wikipedia/full/pagelinks'     -- raw extracted pagelinks
+%default AUGMENTED_PAGELINKS_OUT '/data/results/wikipedia/full/pagelinks'     -- augmented pagelinks
+
+page_metadata = LOAD '$PAGE_METADATA' AS (id:int, namespace:int, title:chararray, 
+  restrictions:chararray, counter:long, is_redirect:int, is_new:int, 
+  random:float, touched:int, page_latest:int, len:int);
+links = LOAD '$EXTRACTED_PAGELINKS' AS (from_id:int, into_namespace:int, into_title:chararray);
+
+first_pass_j = JOIN page_metadata BY id RIGHT OUTER, links BY from_id;
+first_pass   = FOREACH first_pass_j GENERATE 
+  links::from_id AS from_id, page_metadata::namespace AS from_namespace, page_metadata::title AS from_title,
+  links::into_namespace AS into_namespace, links::into_title AS into_title;
+second_pass_j = JOIN page_metadata BY (namespace, title) RIGHT OUTER, first_pass BY (into_namespace, into_title);
+second_pass   = FOREACH second_pass_j GENERATE 
+  first_pass::from_id, page_metadata::id,
+  first_pass::from_namespace, first_pass::from_title, 
+  first_pass::into_namespace, first_pass::into_title;
+STORE second_pass INTO '$AUGMENTED_PAGELINKS_OUT';

data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'wukong'
+require 'wukong/streamer/sql_streamer'
+require 'wukong/streamer/encoding_cleaner'
+
+module PagelinksExtractor
+  class Mapper < Wukong::Streamer::SQLStreamer
+    include Wukong::Streamer::EncodingCleaner
+    columns [:int, :int, :string]
+  end
+end
+
+Wukong::Script.new(PagelinksExtractor::Mapper, nil).run

data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb.old

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'wukong'
+
+load '/home/dlaw/dev/wukong/examples/wikipedia/munging_utils.rb'
+
+module PagelinksToTSV
+  class Mapper < Wukong::Streamer::LineStreamer
+
+    COLUMNS = [:int, :int, :string]
+
+    def initialize
+      @sql_parser = MungingUtils::SQLParser.new(COLUMNS)
+    end
+
+    def process(line, &blk)
+      @sql_parser.parse(line, &blk)
+    end
+  end
+end
+
+# go to town
+Wukong::Script.new(
+  PagelinksToTSV::Mapper,
+  nil
+).run

data/examples/munging/wikipedia/pagelinks/undirect_pagelinks.pig

@@ -0,0 +1,29 @@
+/*
+ * Takes a directed edge list and transforms it into an undirected edge list 
+ * that stores edge direction as metadata.
+ * 
+ * Input table should be of the format (from_id:int, into_id:int ... )
+ *
+ * Output format:
+ * from_id:int, into_id:int, a_into_b:int , b_into_a:int, symmetric:int
+ *
+ * a_into_b, b_into_a, and symmetric are really booleans.
+ */
+
+%default AUGMENTED_PAGELINKS      '/data/results/wikipedia/full/pagelinks'            -- all wikipedia pagelinks (see augment_pagelinks.pig)
+%default UNDIRECTED_PAGELINKS_OUT '/data/results/wikipedia/full/undirected_pagelinks' -- undirected pagelinks
+
+edges = LOAD '$AUGMENTED_PAGELINKS' AS (from:int, into:int);
+edges_sorted = FOREACH edges GENERATE 
+  ((from <= into)? from : into) AS node_a,
+  ((from <= into)? into : from) AS node_b,
+  ((from <= into)? 1 : 0) AS a_to_b,
+  ((from <= into)? 0 : 1) AS b_to_a;
+edges_grouped = GROUP edges_sorted by (node_a, node_b);
+edges_final = FOREACH edges_grouped GENERATE 
+  group.node_a AS node_a,
+  group.node_b AS node_b,
+  ((SUM(edges.$2) > 0) ? 1:0) AS a_into_b,
+  ((SUM(edges.$3) > 0) ? 1:0) AS b_into_a,
+  ((SUM(edges.$2) > 0 AND SUM(edges.$3) > 0) ? 1:0) as symmetric:int;
+STORE edges final INTO '$UNDIRECTED_PAGELINKS_OUT';