ul-wukong 4.1.0

data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+require_relative './dbpedia_common'
+
+# ## Usage
+#
+# Flattens the wikipedia 'enwiki-latest-pages-articles.xml.gz' into a
+# one-line-per-record heap.
+#
+#    examples/munging/wikipedia/dbpedia/extract_geolocations.rb --rm --run \
+#      /data/origin/wikipedia/dbpedia/geo_coordinates_en.nq \
+#      /data/results/wikipedia/dbpedia-geo_coordinates_en.tsv
+#
+
+# ## Sample Pig Schema
+#
+# geolocations = LOAD '$geolocations' AS
+#   (id:long, namespace:int, title:chararray, longitude:float, latitude:float);
+#
+
+module Dbpedia
+  class GeocoordinatesExtractor < Wukong::Streamer::LineStreamer
+    include MungingUtils
+
+    KNOWN_LINE_RE = %r{\A(?:
+         \#\sstarted
+      |  <http://dbpedia\.org/resource/[^>]+>\s
+         <http://(
+            www\.georss\.org/georss/point
+         |  www\.w3\.org/1999/02/22-rdf-syntax-ns\#type
+         |  www\.w3\.org/2003/01/geo/wgs84_pos\#(?:lat|long)
+      )>
+    )}x
+
+    # it's on one line in the actual dataset; split here for readability
+    GEO_RSS_RE = %r{\A
+         <http://dbpedia\.org/resource/(?<title>[^>]+)>
+      \s <http://www\.georss\.org/georss/point>
+      \s "(?<lat>#{DECIMAL_NUM_RE})\s(?<lng>#{DECIMAL_NUM_RE})"@en
+      \s <http://en\.wikipedia\.org/wiki/(?:[^\?]+)\?oldid=(?<article_id>\d+)>
+      \s \.
+    \z}x
+
+    def warn_record(desc, record=nil)
+      record_info = MultiJson.encode(record)[0..1000] rescue "(unencodeable record) #{record.inspect[0..100]}"
+      Log.warn [desc, record_info].join("\t")
+    end
+
+    ARTICLE_NAMESPACE = 0
+
+    # The file is in that godawful semantic web format. Let's fix that.
+    #
+    #     <http://dbpedia.org/resource/Alabama> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.opengis.net/gml/_Feature> <http://en.wikipedia.org/wiki/Alabama?oldid=495507959> .
+    #     <http://dbpedia.org/resource/Alabama> <http://www.w3.org/2003/01/geo/wgs84_pos#lat> "33.0"^^<http://www.w3.org/2001/XMLSchema#float> <http://en.wikipedia.org/wiki/Alabama?oldid=495507959> .
+    #     <http://dbpedia.org/resource/Alabama> <http://www.w3.org/2003/01/geo/wgs84_pos#long> "-86.66666666666667"^^<http://www.w3.org/2001/XMLSchema#float> <http://en.wikipedia.org/wiki/Alabama?oldid=495507959> .
+    #     <http://dbpedia.org/resource/Alabama> <http://www.georss.org/georss/point> "33.0 -86.66666666666667"@en <http://en.wikipedia.org/wiki/Alabama?oldid=495507959> .
+    #
+    # The lines seem to be redundant, with the georss one containing what we need, so just filter for those
+    #
+    def process(line)
+      if not KNOWN_LINE_RE.match(line) then warn_record("Unrecognized line type", line) ; return ; end
+      return unless $1 == 'www.georss.org/georss/point'
+      geo_info = GEO_RSS_RE.match(line)
+      if not geo_info then warn_record("Unrecognized georss line", line) ; return ; end
+
+      result = [
+        geo_info[:article_id],
+        ARTICLE_NAMESPACE, # the dbpedia stuff is all NS 0
+        geo_info[:title],
+        geo_info[:lng],
+        geo_info[:lat],
+      ]
+      yield result
+    end
+  end
+
+end
+
+Wukong::Script.new(Dbpedia::GeocoordinatesExtractor, nil).run

data/examples/munging/wikipedia/dbpedia/extract_links-cruft.rb

@@ -0,0 +1,66 @@
+
+Settings.define :dbpedia_filetype, description: 'The dbpedia file type ("geo_coordinates", etc) -- taken from input filename if available'
+
+# Settings[:dbpedia_filetype] ||= Settings[:input_paths].to_s
+      # Settings[:dbpedia_filetype] = File.basename(Settings[:dbpedia_filetype]).gsub(/[\.\-].*/, '')
+      # @flavor, flavor_info = DBPEDIA_FLAVOR_INFO.detect{|flavor, (filename, _r)| filename == Settings[:dbpedia_filetype] }
+      # @kind, @filename, @regexps = flavor_info
+
+  DBPEDIA_FLAVOR_INFO = {
+    title:               ['labels_en',                               [:title,              ],  ],
+    page_id:             ['page_ids_en',                             [:page_id,            ],  ],
+    wikipedia_link:      ['wikipedia_links_en',                      [:wikipedia_links, :wikipedia_backlink, :wikipedia_lang,      ],   ],
+    abstract_short:      ['short_abstracts_en',                      [:abstract_short,      ],  ],
+    abstract_long:       ['long_abstracts_en',                       [:abstract_long,       ],  ],
+    geo_coordinates:     ['geo_coordinates_en',                      [:geo_coordinates, :geo_coord_skip_a, :geo_coord_skip_b, ],        ],
+    #                                                                #
+    page_links:          ['page_links_unredirected_en',              [:page_links,          ],  ],
+    disambiguations:     ['disambiguations_unredirected_en',         [:disambiguations,     ],  ],
+    redirects:           ['redirects_transitive_en',                 [:redirects,           ],  ],
+    #                                                                #
+    external_links:      ['external_links_en',                       [:external_links,      ],  ],
+    homepages:           ['homepages_en',                            [:homepages,           ],  ],
+    geonames:            ['geonames_links',                          [:geonames,            ],  ],
+    musicbrainz:         ['musicbrainz_links',                       [:musicbrainz,         ],  ],
+    nytimes:             ['nytimes_links',                           [:nytimes,             ],  ],
+    uscensus:            ['uscensus_links',                          [:uscensus,            ],  ],
+    pnd:                 ['pnd_en',                                  [:pnd,                 ],  ],
+    #                                                                #
+    article_categories:  ['article_categories_en',                   [:article_categories,  ],  ],
+    category_title:      ['category_labels_en',                      [:title,     ],  ],
+    category_skos:       ['skos_categories_en',                      [:category_skos_skip, :category_skos_title, :category_skos_reln ],      ],
+    #                                                                #
+    wordnet:             ['wordnet_links',                           [:wordnet,             ],  ],
+    persondata:          ['persondata_unredirected_en',              [:persondata_reln, :persondata_type,          ],   ],
+    yago:                ['yago_links',                              [:yago,                :instance_type_a, :instance_type_b,  ],  ],
+    instance_types:      ['instance_types_en',                       [:yago,                :instance_type_a, :instance_type_b,  ],  ],
+    property_specmap:    ['specific_mappingbased_properties_en',     [:property_specmap,    ],  ],
+    property_mapped:     ['mappingbased_properties_unredirected_en', [
+        :property_str, :property_bool, :property_int,
+        :property_float, :property_date, :property_yearmonth, :property_monthday,
+        :persondata_reln, :persondata_type, :property_foaf, :property_desc, ],      ],
+    topical_concepts:    ['topical_concepts_unredirected_en',        [:topical_concepts,    ],  ],
+  }
+
+module Re
+  ##
+  # Container for the character classes specified in
+  # <a href="http://www.ietf.org/rfc/rfc3986.txt">RFC 3986</a>.
+  # Borrowed from the addressable gem
+  module Uri
+    ALPHA      = "a-zA-Z"
+    DIGIT      = "0-9"
+    GEN_DELIMS = "\\:\\/\\?\\#\\[\\]\\@"
+    SUB_DELIMS = "\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\="
+    RESERVED   = GEN_DELIMS + SUB_DELIMS
+    UNRESERVED = ALPHA + DIGIT + "\\-\\.\\_\\~"
+    PCHAR      = UNRESERVED + SUB_DELIMS + "\\:\\@"
+    SCHEME     = ALPHA + DIGIT + "\\-\\+\\."
+    AUTHORITY  = PCHAR
+    PATH       = PCHAR + "\\/"
+    QUERY      = PCHAR + "\\/\\?"
+    FRAGMENT   = PCHAR + "\\/\\?"
+    #
+    PATHSEG    = ""
+  end
+end

data/examples/munging/wikipedia/dbpedia/extract_links.rb

@@ -0,0 +1,260 @@
+#!/usr/bin/env ruby
+require_relative './dbpedia_common'
+require 'ap'
+
+# Notes:
+#
+# * disambiguation: `generic disambiguates specifics` -- `["Alien", "Alien_(law)"]` and `["Alien", "Alien_(film)"]`
+# * redirects:      `dupe    redirects to  actual`    -- `["Oxygen-13", "Isotopes_of_oxygen"]`
+# * page_link:      `from    links to      into`      -- `["Achilles", "Greeks"]
+
+module Dbpedia
+
+  DECIMAL_NUM_RE  = '[\-\+\d]+\.\d+'
+  URI_PATHCHARS   = '\w\-\.\'~!$&()*+,;=:@'
+  # all backslash-escaped character, or non-quotes, up to first quote
+  DBLQ_STRING_C   = '"(?<%s>(?:\\.|[^\"])*)"'
+
+  # output flavors:
+  #
+  # :abstract_long   :abstract_short      :category :category_reln  :disambiguation
+  # :external_link   :geo_coordinates     :homepage :instance_of    :page_id :page_link
+  # :persondata_reln :property :redirects :sameas   :subject :title :wikipedia_link
+  #
+
+  MAPPING_INFO = {
+    # atomic topic properties
+    title:               { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,  :title,                                      ],  },
+    page_id:             { kind: :page_id,             fields: [:page_id, :wp_ns, :wikipedia_id, :wikipedia_pageid,                           ],  },
+    abstract_short:      { kind: :abstract_short,      fields: [:page_id, :wp_ns, :wikipedia_id, :abstract,                                   ],  },
+    abstract_long:       { kind: :abstract_long,       fields: [:page_id, :wp_ns, :wikipedia_id, :abstract,                                   ],  },
+    wikipedia_lang:      { kind: :skip,                fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :url,     :slug,  :lang,                  ],  },
+    wikipedia_link:      { kind: :wikipedia_link,      fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :url,     :slug,  :revision_id,           ],  },
+    wikipedia_backlink:  { kind: :skip,                fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :url,     :slug,  :revision_id,           ],  },
+    geo_coordinates:     { kind: :geo_coordinates,     fields: [:page_id, :wp_ns, :wikipedia_id, :lat,        :lng,                           ],  },
+    geo_coord_skip_a:    { kind: :skip,                fields: [],      },
+    geo_coord_skip_b:    { kind: :skip,                fields: [],      },
+    # links between topics
+    page_link:           { kind: :page_link,           fields: [:page_id, :wp_ns, :from_id,      :relation,   :into_id,                       ],  },
+    disambiguation:      { kind: :disambiguation,      fields: [:page_id, :wp_ns, :generic_wpid, :relation,   :specific_wpid,                 ],  },
+    redirects:           { kind: :redirects,           fields: [:page_id, :wp_ns, :dupe_id,      :relation,   :wikipedia_id,                  ],  },
+    # external links and sameas'es
+    external_link:       { kind: :external_link,       fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :weblink_url,                   ],  },
+    homepage:            { kind: :homepage,            fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :weblink_url,                   ],  },
+    geonames:            { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :geonames_id,                   ],  },
+    musicbrainz:         { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :musicbrainz_type,  :musicbrainz_id,], },
+    nytimes:             { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :nytimes_id,                    ],  },
+    pnd:                 { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :pnd_id,                        ],  },
+    uscensus:            { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :country_id, :state_id, :kind, :adm2_id, :adm3_id, :adm4_id],  },
+    # category links
+    category_skos_type:  { kind: :instance_of,         fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :obj_class                      ],  },
+    category_skos_title: { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :val_type,   :category_title,   ],  },
+    category:            { kind: :category,            fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :specific_wpid,                 ],  },
+    category_subject:    { kind: :subject,             fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :into_wpid,                     ],  },
+    category_reln:       { kind: :category_reln,       fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :into_wpid,                     ],  },
+    # properties
+    wordnet:             { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :wn_reln,    :wn_class,   :wn_pos, :wn_idx,  ],  },
+    property_bool:       { kind: :property_bool,       fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_int:        { kind: :property_int,        fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_float:      { kind: :property_float,      fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_date:       { kind: :property_date,       fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_yearmonth:  { kind: :property_yearmonth,  fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_monthday:   { kind: :property_monthday,   fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_str:        { kind: :property_str,        fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    #
+    persondata_reln:     { kind: :persondata_reln,     fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :into_wpid,                     ],  },
+    # persondata_type:   { kind: :# persondata_type,   fields: [:page_id, :wp_ns, :wikipedia_id, :property,                                   ],  },
+    property_foaf:       { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_desc:       { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :name,             ],  },
+    yago:                { kind: :category,            fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :obj_class,                     ],  },
+    instance_type_a:     { kind: :instance_of,         fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :obj_class,                     ],  },
+    instance_type_b:     { kind: :instance_of,         fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :obj_class,                     ],  },
+    property_specmap:    { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :units,        :val,            ],  },
+    # topical_concepts:  { kind: :# topical_concepts,  fields: [:page_id, :wp_ns, :wikipedia_id, :skos_subject   :x,                          ],  },
+  }
+
+  RDF_RES = {
+    # type descriptions
+    dbpedia_class:    'http://dbpedia\.org/class/(?<%s>[^>\s]+)',
+    dbpedia_ontb:     'http://dbpedia\.org/ontology',
+    dbpedia_ont:      'http://dbpedia\.org/ontology/(?<%s>[\w\/]+)',
+    dbpedia_prop:     'http://dbpedia\.org/property/(?<%s>\w+)',
+    dbpedia_rsrc:     'http://dbpedia\.org/resource/(?<%s>['           + URI_PATHCHARS + '%%\/]+)',
+    yago_class:       'http://dbpedia\.org/class/(?<%s>yago)/(?<%s>['  + URI_PATHCHARS + '%%\/]+)',
+    wikipedia_rsrc:   '(?<%s>http://\w\w\.wikipedia\.org/wiki/(?<%s>[' + URI_PATHCHARS + '%%\/]+))',
+    wiki_category:    'http://en\.wikipedia\.org/wiki/Category:Futurama?oldid=485425712\\#absolute-line=1',
+    wiki_link_id:     'http://en\.wikipedia\.org/wiki/(?<%s>[^\?]+)\?oldid=(?<%s>\d+)(?:\\#absolute-line=(?<%s>\d+))?',
+    wiki_link_id_sec: 'http://en\.wikipedia\.org/wiki/(?<%s>[^\?]+)\?oldid=(?<%s>\d+)\\#?(?:section=(?<%s>.*?)\&relative-line=(?<%s>\d+))?(?:&?absolute-line=(?<%s>\d+))?',
+    purl_desc:        'http://purl\.org/dc/elements/1\.1/(?<%s>description)',
+    purl_lang:        'http://purl\.org/dc/elements/1\.1/language',
+    purl_subject:     'http://purl\.org/dc/terms/subject',
+    rdf_type:         'http://www\.w3\.org/1999/02/22-rdf-syntax-ns\\#type',
+    rdf_comment:      'http://www\.w3\.org/2000/01/rdf-schema\\#comment',
+    rdf_label:        'http://www\.w3\.org/2000/01/rdf-schema\\#label',
+    # external links and sameas'es
+    same_as:          'http://www\.w3\.org/2002/07/owl\\#sameAs',
+    wordnet_inst:     'http://www\.w3\.org/2006/03/wn/wn20/instances/(?<%s>synset)-(?<%s>\w+)-(?<%s>noun)-(?<%s>[0-9]+)',
+    musicbrainz_rsrc: 'http://zitgist\.com/music/(?<%s>\w+)/(?<%s>[a-f0-9\-]+)',
+    nytimes_rsrc:     'http://data\.nytimes\.com/(?<%s>[A-Z0-9]+)',
+    geonames_rsrc:    'http://sws\.geonames\.org/(?<%s>\d+)/',
+    georss_type:      'http://www\.georss\.org/georss/point',
+    wgs_latorlng:     'http://www\.w3\.org/2003/01/geo/wgs84_pos\\#(?:lat|long)',
+    #                  http://www.rdfabout.com/rdf/usgov/geo/  us     /     ak       /    counties   /bethel_area  /an_subarea   /aniak          >
+    uscensus_url:     'http://www.rdfabout.com/rdf/usgov/geo/(?<%s>us)/(?<%s>\w\w)(?:/(?<%s>counties)/(?<%s>\w+)(?:/(?<%s>\w+)\/?(?<%s>\w+)?)?)?',
+    # category links
+    skos_subject:     'http://www\.w3\.org/2004/02/skos/core\\#subject',
+    skos_concept:     'http://www\.w3\.org/2004/02/skos/core\\#(?<%s>[a-zA-Z]+)',
+    foaf_homepage:    'http://xmlns\.com/foaf/0\.1/homepage',
+    foaf_name:        'http://xmlns\.com/foaf/0\.1/name',
+    foaf_topic:       'http://xmlns\.com/foaf/0\.1/(?:isPrimaryTopicOf|primaryTopic)',
+    foaf_prop:        'http://xmlns\.com/foaf/0\.1/(?<property>\w+)',
+    # property values
+    georss_latlng:    '\"(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\\s(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\"@\w\w',
+    rdf_eol:          '\\.',
+    #
+    rdf_bool:         '\"(?<%s>true|false                        )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>boolean)>',
+    rdf_date:         '\"(?<%s>-?\d\d\d\d-\d\d-\d\d              )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>date)>',
+    rdf_yearmonth:    '\"(?<%s>-?\d\d\d\d-\d\d                   )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>gYearMonth)>',
+    rdf_monthday:     '\"(?<%s>--\d\d-\d\d                       )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>gMonthDay)>',
+    rdf_int:          '\"(?<%s>[\+\-]?\d+                        )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>integer|gYear|positiveInteger|nonNegativeInteger)>',
+    rdf_float:        '\"(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>float|double)>',
+    # all backslash-escaped character, or non-quotes, up to first quote
+    rdf_string:       '"(?<%s>(?:\\\\.|[^\"])*)"@en',
+    dbpedia_value:    '"(?<%s>(?:\\\\.|[^\"])*)"\\^\\^<http://dbpedia\.org/datatype/(?<%s>[a-zA-Z]+)>',
+    #
+    url_loose:        '(?<%s>(?:https?|ftp)://(?:[a-zA-Z0-9\-]+\.)+(?:[a-zA-Z\-]+)[^\s>]*)',
+    # rdf_value:        '\"(?<%s>-?\d\d\d\d-\d\d-\d\d|-?\d\d\d\d-\d\d|--\d\d-\d\d|[\+\-]?\d+|[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?|true|false)\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>integer|date|gYearMonth|gMonthDay|gYear|positiveInteger|nonNegativeInteger|float|double|boolean)>',
+    schema_type:      'http://(?<%s>www\\.w3\\.org/2002/07/owl|schema\\.org|dbpedia\\.org/ontology|purl\\.org/ontology|xmlns.com/foaf/0\\.1)[/\\#](?<%s>[^>]+)'
+  }
+
+  SCHEMA_SCHEMES = {
+    'www.w3.org/2002/07/owl' => 'owl',
+    'schema.org'             => 'schemaorg',
+    'dbpedia.org/ontology'   => 'dbpedia',
+    'purl.org/ontology'      => 'purl',
+    'xmlns.com/foaf/0.1'     => 'foaf'
+  }
+
+  # lookup regexp in above table, sub in variable names
+  private
+  def self.r(regexp_name, *args)
+    RDF_RES[regexp_name] % args
+  end
+  public
+
+  MAPPING_RES = {
+    # atomic topic properties
+    title:               %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_label)}>                          \s#{r(:rdf_string, :title )}                   \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}x,
+    page_id:             %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/wikiPageID>            \s#{r(:rdf_int,    :wikipedia_pageid, :_dtyp)} \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_lang:      %r{\A<#{r(:wikipedia_rsrc,  :url, :slug)}>         \s<#{r(:purl_lang)}>                          \s#{r(:rdf_string, :lang)}                       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_link:      %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:foaf_topic)}>                         \s<#{r(:wikipedia_rsrc, :url, :slug)}>         \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_backlink:  %r{\A<#{r(:wikipedia_rsrc,  :url, :slug)}>        \s<#{r(:foaf_topic)}>                         \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    abstract_short:      %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_comment)}>                        \s#{r(:rdf_string, :abstract)}                 \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}x,
+    abstract_long:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/abstract>              \s#{r(:rdf_string, :abstract)}                 \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}xm,
+    geo_coordinates:     %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:georss_type)}>                        \s#{r(:georss_latlng, :lat, :lng)}             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    geo_coord_skip_a:    %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<http://www\.opengis\.net/gml/_Feature>      \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    geo_coord_skip_b:    %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:wgs_latorlng)}>                       \s#{r(:rdf_float, :val, :_dtyp)}               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    # links between topic
+    page_link:           %r{\A<#{r(:dbpedia_rsrc,    :from_id)}>              \s<#{r(:dbpedia_ontb)}/wikiPageWikiLink>      \s<#{r(:dbpedia_rsrc, :into_id)}>              \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    disambiguation:      %r{\A<#{r(:dbpedia_rsrc,    :generic_wpid)}>         \s<#{r(:dbpedia_ontb)}/wikiPageDisambiguates> \s<#{r(:dbpedia_rsrc, :specific_wpid)}>        \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    redirects:           %r{\A<#{r(:dbpedia_rsrc,    :dupe_id)}>              \s<#{r(:dbpedia_ontb)}/wikiPageRedirects>     \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>                                                                                                                       \s#{r(:rdf_eol)}   \z}x,
+    # external links and sameas'es
+    external_link:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/wikiPageExternalLink>  \s<#{r(:url_loose, :weblink_url)}>             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    homepage:            %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:foaf_homepage)}>                      \s<#{r(:url_loose, :weblink_url)}>             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    geonames:            %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:geonames_rsrc, :geonames_id)}>                                                                                                                       \s#{r(:rdf_eol)}  \z}x,
+    musicbrainz:         %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:musicbrainz_rsrc, :musicbrainz_type, :musicbrainz_id)}>                                                                                              \s#{r(:rdf_eol)}  \z}x,
+    nytimes:             %r{\A<#{r(:nytimes_rsrc,    :nytimes_id)}>           \s<#{r(:same_as)}>                            \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>                                                                                                                       \s#{r(:rdf_eol)}   \z}x,
+    uscensus:            %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:uscensus_url, :country_id, :state_id, :kind, :adm2_id, :adm3_id, :adm4_id)}>                                                                         \s#{r(:rdf_eol)}  \z}x,
+    pnd:                 %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/individualisedPnd>     \s#{r(:rdf_string, :pnd_id)}                   \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    # category links
+    category:            %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:purl_subject)}>                       \s<#{r(:dbpedia_rsrc, :specific_wpid)}>        \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    category_skos_type:  %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:skos_concept, :obj_class)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    category_subject:    %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:skos_subject, :relation)}>            \s<#{r(:dbpedia_rsrc, :into_wpid)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    category_reln:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:skos_concept, :relation)}>            \s<#{r(:dbpedia_rsrc, :into_wpid)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    category_skos_title: %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:skos_concept, :relation)}>            \s#{r(:rdf_string, :category_title)}           \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}x,
+    # properties
+    wordnet:             %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_prop, :property)}>            \s<#{r(:wordnet_inst, :wn_reln, :wn_class, :wn_pos, :wn_idx)}>                                                                                               \s#{r(:rdf_eol)}  \z}x,
+    #
+    property_bool:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_bool,      :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_int:        %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_int,       :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_float:      %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_float,     :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_date:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_date,      :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_yearmonth:  %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_yearmonth, :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_monthday:   %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_monthday,  :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_str:        %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_string,    :val)            }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    #
+    persondata_reln:     %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s<#{r(:dbpedia_rsrc, :into_wpid)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_foaf:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:foaf_prop, :property)}>               \s#{r(:rdf_string, :val)}                      \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_desc:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:purl_desc, :property)}>               \s#{r(:rdf_string,:name)}                      \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    yago:                %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:yago_class,  :scheme, :obj_class)}>                                                                                                                  \s#{r(:rdf_eol)}  \z}x,
+    instance_type_a:     %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:dbpedia_ont,          :obj_class)}>    \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    instance_type_b:     %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:schema_type, :org,    :obj_class)}>    \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_specmap:    %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:dbpedia_value, :val, :units)}           \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    # topical_concepts:  %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:skos_subject)}>                       \s<#{r(:x, )}> \z},
+  }
+  MAPPING_RES.each{|re_name, re| MAPPING_INFO[re_name][:re] = re }
+  SKIPPAPLE_FIELDS = [:flavor, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno, :val_lang, :name_lang, :_dtyp]
+
+  class RdfExtractor < Wukong::Streamer::LineStreamer
+    include MungingUtils
+    attr_accessor :flavor, :kind, :filename, :regexps, :seen_keys, :seen_props
+
+    def initialize(*args)
+      @seen_keys  = Hash.new(0)
+      @seen_props = Hash.new(0)
+    end
+
+    def record_for_flavor(kind, fields, flavor, hsh)
+      hsh.merge!( wp_ns: 0, flavor: flavor )
+      return if kind == :skip
+
+      case flavor
+      when :property_str, :property_foaf   then  hsh[:val] = MultiJson.encode(hsh[:val])
+      when :abstract_long, :abstract_short then  hsh[:abstract] = MultiJson.encode(hsh[:abstract])
+      when :title                          then  hsh[:title] = MultiJson.encode(hsh[:title]) ; hsh[:property] = 'title'
+      when :category_skos_title            then  hsh[:category_title] = MultiJson.encode(hsh[:category_title])
+      when :category_skos_type then hsh[:scheme] = 'skos'
+      when :category_subject   then hsh[:scheme] = 'subject'
+      when :instance_type_a    then hsh[:scheme] = 'dbpedia'
+      when :instance_type_b
+        hsh[:scheme] = SCHEMA_SCHEMES[hsh.delete(:org)]
+        return if hsh[:scheme] == 'owl'
+      when :wikipedia_link, :wikipedia_backlink
+        raise "Titles disagree!" unless hsh[:slug] == hsh[:wikipedia_id]
+      end
+
+      # record seen properties, seen fields
+      hsh.except(*fields).except(*SKIPPAPLE_FIELDS).
+        each{|key, val| @seen_keys[key] += 1 if val.present? }
+      seen_props[hsh[:property]] += 1 if hsh[:property].present?
+      sanity_check(hsh)
+      #
+      [kind] + hsh.values_at(*fields)
+    end
+
+    def sanity_check(hsh)
+      hsh.each{|key,val| raise if CONTROL_CHARS_RE =~ val.to_s }
+    end
+
+    def after_stream
+      Log.info ["seen keys:", seen_keys.inspect, "seen props:", seen_props.inspect].join("\t")
+    end
+
+    def process(line)
+      return if line =~ /\A(?:\#|$)/
+      if (line =~ /=> \w+\.\w+ <=/) then yield [line] ; return ; end
+
+      MAPPING_INFO.each do |flavor, info|
+        next unless mm = info[:re].match(line)
+        yield record_for_flavor(info[:kind], info[:fields], flavor, mm.as_hash)
+        return
+      end
+
+      Log.warn ['not found:', line].join("\t")
+    end
+  end
+end
+
+
+
+Wukong::Script.new(Dbpedia::RdfExtractor, nil).run

data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb

@@ -0,0 +1,20 @@
+module Dbpedia
+
+  class SameasExtractor < Wukong::Streamer::LineStreamer
+    include MungingUtils
+
+    SAME_AS_RE = %r{\A
+         <http://dbpedia\.org/resource/(?<title>[^>]+)>
+      \s <http://www\.w3\.org/2002/07/owl\#sameAs>
+      \s <http://(?<target>[^>]+)>
+      \s \.
+    \z}x
+
+    def recordize(line)
+      same_as = SAME_AS_RE.match(line)
+      if not same_as then warn_record("Unrecognized line type", line) ; return end
+      [same_as[:title], same_as[:target]]
+    end
+  end
+
+end

data/examples/rake_helper.rb

@@ -0,0 +1,97 @@
+require 'configliere' ; Settings.use :commandline
+require 'gorillib'
+require 'gorillib/data_munging'
+require 'pry'
+require 'rake'
+
+<<<<<<< HEAD
+PROJ_ROOT = (ENV['PROJ_CONTENTS'] || File.expand_path('..', File.dirname(__FILE__)))
+=======
+BOOK_ROOT = (ENV['BOOK_CONTENTS'] || File.expand_path('..', File.dirname(__FILE__)))
+>>>>>>> produce_consume
+
+Settings.define :mini, type: :boolean, default: false, description: "use sample data or full data?"
+Settings.resolve!
+Settings[:mini_slug] = Settings.mini ? "-sample" : ""
+
+# dummy dependency to force a task. TASK FORCE DELTA GO
+task :force
+
+Pathname.register_paths(
+<<<<<<< HEAD
+  proj_root: PROJ_ROOT,
+  root: [:proj_root],
+  code: [:proj_root, 'code'],
+  data: [:proj_root, 'data'],
+  work: [:proj_root, 'tmp'],
+=======
+  book_root: BOOK_ROOT,
+  root: [:book_root],
+  code: [:book_root, 'code'],
+  data: [:book_root, 'data'],
+  work: [:book_root, 'tmp'],
+>>>>>>> produce_consume
+  )
+
+require 'rake/name_space'
+module ::Rake
+  class NameSpace
+    def name
+      @scope.join(':')
+    end
+
+    def direct_tasks
+      tasks.find_all{|task| task.name =~ /\A#{name}:\w+\z/ }
+    end
+  end
+end
+
+def step(target, options)
+  deps       = [options[:after]].flatten.compact
+  Array.wrap(options[:invoke]).each{|task_name| Rake::Task[task_name].invoke }
+  desc(options[:doc]) if options[:doc]
+  task target => deps do
+    yield target if block_given?
+  end
+  target
+end
+
+def file_task(name, options={})
+  target     = Pathname.of(name)
+  target_dir = File.dirname(target.to_s)
+  task(name => target)
+  #
+  directory(target_dir)
+  deps       = [options[:after], target_dir].flatten.compact
+  file target => deps do
+    Array.wrap(options[:invoke]).each{|task_name| Rake::Task[task_name].invoke }
+    Log.info "Creating #{name} => #{target}"
+    yield target if block_given?
+  end
+  target
+end
+
+def create_file(name, options={})
+  file_task(name, options) do |target|
+    File.open(target, 'wb') do |target_file|
+      yield target_file
+    end
+  end
+end
+
+# * accumulates all symbol-named tasks that are direct children
+#
+# @example Will make task parse depending on 'parse:bob'
+#   chain('parse') do
+#     task('bob'){ ... }
+#     chain('nest'){ task('two_down') }
+#   end
+#
+def chain(name, doc=nil, &block)
+  desc(doc) if doc
+  task(name)
+  return unless block
+  ns = namespace(name, &block)
+  task(name => ns.direct_tasks)
+  ns
+end