wukong 3.0.0.pre2 → 3.0.0.pre3

data/examples/munging/airline_flights/indexable.rb

@@ -0,0 +1,75 @@
+module Gorillib
+  module Model
+
+    #
+    # @example
+    #   class Airport
+    #     include Gorillib::Model::Indexable
+    #     # ... define model
+    #     index_on :icao_code, :city
+    #   end
+    #   Airport.for_icao_code('KAUS') #=> #<Airport icao_code="KAUS" ... >
+    #   Airport.for_city('Austin')    #=> #<Airport icao_code="KAUS" ... >
+    #
+    # You must implement a `load` method
+    #
+    module Indexable
+      extend Gorillib::Concern
+
+      included do |base|
+        base.class_attribute :lookups, instance_writer: false
+        self.lookups ||= []
+      end
+
+      module ClassMethods
+
+        def values
+          @values ||= Array.new
+        end
+
+        def flush_lookups
+          lookups.each{|idx| remove_instance_variable("@#{idx}") if instance_variable_defined?("@#{idx}") }
+        end
+
+        #
+        # @example
+        #   class Airport
+        #     index_on :icao_code, [:city, :cities]
+        #   end
+        #   Airport.for_icao_code('KAUS') #=> #<Airport icao_code="KAUS" ... >
+        #   Airport.for_city('Austin')    #=> #<Airport icao_code="KAUS" ... >
+        #
+        # NOTE: `.#{key_name}_index` method is NOT part of the framework interface.
+        # only the `.for_#{keyname}` method is suported.
+        def index_on(*key_names)
+          self.lookups += key_names
+          self.lookups.uniq!
+          #
+          key_names.each do |key_name, index_name|
+            index_name ||= "#{key_name}_index"
+            class_eval <<-EOV, __FILE__, __LINE__ + 1
+              class << self
+                def #{index_name}                                # def name_index
+                  @#{index_name} ||=                             #   @name_index ||=
+                    Hash[values.map{|el| [el.#{key_name}, el] }] #     Hash[values.map{|el| [el.name, el]
+                end                                              # end
+                protected(:#{index_name})                        # protected :name_index
+              end
+            EOV
+
+            instance_eval <<-EOV, __FILE__, __LINE__ + 1
+              def for_#{key_name}(*args, &block)               # def for_name(*args, &block)
+                #{index_name}.fetch(*args, &block)             #   name_index.fetch(*args, &block)
+              end                                              # end
+            EOV
+          end
+        end
+
+
+
+      end
+
+    end
+
+  end
+end

data/examples/munging/airline_flights/indexable_spec.rb

@@ -0,0 +1,90 @@
+require 'spec_helper'
+require 'gorillib/model'
+require 'gorillib/pathname'
+#
+require 'gorillib/model/serialization'
+require 'gorillib/model/serialization/tsv'
+require 'gorillib/array/hashify'
+#
+require 'wu/model/indexable'
+
+describe Gorillib::Model::Indexable, :model_spec, :only do
+  let(:mock_array){ mock('array') }
+
+  let(:country_code_class) do
+    module Gorillib::Test
+      remove_const(:CountryCode) if defined?(CountryCode)
+
+      class CountryCode
+        include Gorillib::Model
+        include Gorillib::Model::Indexable
+        field :alpha_2_code,     String, position: 0
+        field :name,             String, position: 1
+        def self.load
+          self.values << new('dj', 'Djibouti')
+          self.values << new('us', 'United States of America')
+          values
+        end
+      end
+
+    end
+    Gorillib::Test::CountryCode
+  end
+
+  let(:djibouti){ country_code_class.new('dj', 'Djibouti') }
+  let(:usa     ){ country_code_class.new('us', 'United States of America') }
+
+  context 'test setup' do
+    subject{ country_code_class.load }
+    it{ should == [djibouti, usa] }
+  end
+
+  context '.values' do
+    # before{ country_code_class.send(:remove_instance_variable, '@values') }
+    it 'gets its values from .load' do
+      country_code_class.should_receive(:load).once.and_return mock_array
+      country_code_class.values.should equal(mock_array)
+    end
+    it 'memoizes once it is called' do
+      country_code_class.should_receive(:load).once.and_return mock_array
+      country_code_class.values.should equal(mock_array)
+      country_code_class.values.should equal(mock_array)
+    end
+  end
+
+  context '.index_on' do
+    it 'defines a .for_foo method' do
+      country_code_class.should_not respond_to(:for_name)
+      country_code_class.index_on(:name)
+      country_code_class.should respond_to(:for_name)
+      country_code_class.protected_methods.should include(:name_index)
+    end
+  end
+
+  context '.for_foo' do
+    before{ country_code_class.index_on :name }
+    context 'behaves like Hash#fetch:' do
+      context 'when key is not present' do
+        it 'retrieves a value if in the index' do
+          country_code_class.for_name('Djibouti').should == djibouti
+        end
+      end
+      context 'when key is not present' do
+        it 'and no default it raises KeyError' do
+          expect{ country_code_class.for_name('Yo Mama') }.to raise_error(KeyError, 'key not found: "Yo Mama"')
+        end
+        it 'returns default value if given' do
+          yo_mama = country_code_class.for_name('Yo Mama', 'wears combat boots')
+          yo_mama.should == 'wears combat boots'
+        end
+        it 'calls block if given' do
+          she = nil
+          so_fat = country_code_class.for_name('Yo Mama'){ she = 'sits around the house' ; 'when she sits' }
+          so_fat.should == 'when she sits'
+          she.should    == 'sits around the house'
+        end
+      end
+    end
+  end
+
+end

data/examples/munging/geo/geonames_models.rb

@@ -75,4 +75,33 @@ module Geo
   class GeonamesCountry < GeonamesPlace
     self.place_klass = Geo::Country
   end
+
+  # http://download.geonames.org/export/zip/
+  #
+  # country code      : iso country code, 2 characters
+  # postal code       : varchar(20)
+  # place name        : varchar(180)
+  # admin name1       : 1. order subdivision (state) varchar(100)
+  # admin code1       : 1. order subdivision (state) varchar(20)
+  # admin name2       : 2. order subdivision (county/province) varchar(100)
+  # admin code2       : 2. order subdivision (county/province) varchar(20)
+  # admin name3       : 3. order subdivision (community) varchar(100)
+  # admin code3       : 3. order subdivision (community) varchar(20)
+  # latitude          : estimated latitude (wgs84)
+  # longitude         : estimated longitude (wgs84)
+  # accuracy          : accuracy of lat/lng from 1=estimated to 6=centroid
+  class GeonamesPostal
+    field :country_id,   String, doc: "iso country code, 2 characters"
+    field :postal_id,    String, doc: "varchar(20)"
+    field :name,         String, doc: "varchar(180)"
+    field :admin1_name,  String, doc: "1. order subdivision (state) varchar(100)"
+    field :admin1_id,    String, doc: "1. order subdivision (state) varchar(20)"
+    field :admin2_name,  String, doc: "2. order subdivision (county/province) varchar(100)"
+    field :admin2_id,    String, doc: "2. order subdivision (county/province) varchar(20)"
+    field :admin3_name,  String, doc: "3. order subdivision (community) varchar(100)"
+    field :admin3_id,    String, doc: "3. order subdivision (community) varchar(20)"
+    field :latitude,     String, doc: "estimated latitude (wgs84)"
+    field :longitude,    String, doc: "estimated longitude (wgs84)"
+    field :accuracy,     String, doc: "accuracy of lat/lng from 1=estimated to 6=centroid"
+  end
 end

data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb

@@ -1,4 +1,5 @@
 
 require 'wukong'
 require 'multi_json'
+require 'gorillib'
 require_relative '../utils/munging_utils.rb'

data/examples/munging/wikipedia/dbpedia/extract_links-cruft.rb

@@ -0,0 +1,66 @@
+
+Settings.define :dbpedia_filetype, description: 'The dbpedia file type ("geo_coordinates", etc) -- taken from input filename if available'
+
+# Settings[:dbpedia_filetype] ||= Settings[:input_paths].to_s
+      # Settings[:dbpedia_filetype] = File.basename(Settings[:dbpedia_filetype]).gsub(/[\.\-].*/, '')
+      # @flavor, flavor_info = DBPEDIA_FLAVOR_INFO.detect{|flavor, (filename, _r)| filename == Settings[:dbpedia_filetype] }
+      # @kind, @filename, @regexps = flavor_info
+
+  DBPEDIA_FLAVOR_INFO = {
+    title:               ['labels_en',                               [:title,              ],  ],
+    page_id:             ['page_ids_en',                             [:page_id,            ],  ],
+    wikipedia_link:      ['wikipedia_links_en',                      [:wikipedia_links, :wikipedia_backlink, :wikipedia_lang,      ],   ],
+    abstract_short:      ['short_abstracts_en',                      [:abstract_short,      ],  ],
+    abstract_long:       ['long_abstracts_en',                       [:abstract_long,       ],  ],
+    geo_coordinates:     ['geo_coordinates_en',                      [:geo_coordinates, :geo_coord_skip_a, :geo_coord_skip_b, ],        ],
+    #                                                                #
+    page_links:          ['page_links_unredirected_en',              [:page_links,          ],  ],
+    disambiguations:     ['disambiguations_unredirected_en',         [:disambiguations,     ],  ],
+    redirects:           ['redirects_transitive_en',                 [:redirects,           ],  ],
+    #                                                                #
+    external_links:      ['external_links_en',                       [:external_links,      ],  ],
+    homepages:           ['homepages_en',                            [:homepages,           ],  ],
+    geonames:            ['geonames_links',                          [:geonames,            ],  ],
+    musicbrainz:         ['musicbrainz_links',                       [:musicbrainz,         ],  ],
+    nytimes:             ['nytimes_links',                           [:nytimes,             ],  ],
+    uscensus:            ['uscensus_links',                          [:uscensus,            ],  ],
+    pnd:                 ['pnd_en',                                  [:pnd,                 ],  ],
+    #                                                                #
+    article_categories:  ['article_categories_en',                   [:article_categories,  ],  ],
+    category_title:      ['category_labels_en',                      [:title,     ],  ],
+    category_skos:       ['skos_categories_en',                      [:category_skos_skip, :category_skos_title, :category_skos_reln ],      ],
+    #                                                                #
+    wordnet:             ['wordnet_links',                           [:wordnet,             ],  ],
+    persondata:          ['persondata_unredirected_en',              [:persondata_reln, :persondata_type,          ],   ],
+    yago:                ['yago_links',                              [:yago,                :instance_type_a, :instance_type_b,  ],  ],
+    instance_types:      ['instance_types_en',                       [:yago,                :instance_type_a, :instance_type_b,  ],  ],
+    property_specmap:    ['specific_mappingbased_properties_en',     [:property_specmap,    ],  ],
+    property_mapped:     ['mappingbased_properties_unredirected_en', [
+        :property_str, :property_bool, :property_int,
+        :property_float, :property_date, :property_yearmonth, :property_monthday,
+        :persondata_reln, :persondata_type, :property_foaf, :property_desc, ],      ],
+    topical_concepts:    ['topical_concepts_unredirected_en',        [:topical_concepts,    ],  ],
+  }
+
+module Re
+  ##
+  # Container for the character classes specified in
+  # <a href="http://www.ietf.org/rfc/rfc3986.txt">RFC 3986</a>.
+  # Borrowed from the addressable gem
+  module Uri
+    ALPHA      = "a-zA-Z"
+    DIGIT      = "0-9"
+    GEN_DELIMS = "\\:\\/\\?\\#\\[\\]\\@"
+    SUB_DELIMS = "\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\="
+    RESERVED   = GEN_DELIMS + SUB_DELIMS
+    UNRESERVED = ALPHA + DIGIT + "\\-\\.\\_\\~"
+    PCHAR      = UNRESERVED + SUB_DELIMS + "\\:\\@"
+    SCHEME     = ALPHA + DIGIT + "\\-\\+\\."
+    AUTHORITY  = PCHAR
+    PATH       = PCHAR + "\\/"
+    QUERY      = PCHAR + "\\/\\?"
+    FRAGMENT   = PCHAR + "\\/\\?"
+    #
+    PATHSEG    = ""
+  end
+end

data/examples/munging/wikipedia/dbpedia/extract_links.rb

@@ -2,192 +2,259 @@
 require_relative './dbpedia_common'
 require 'ap'
 
-module Re
-  ##
-  # Container for the character classes specified in
-  # <a href="http://www.ietf.org/rfc/rfc3986.txt">RFC 3986</a>.
-  # Borrowed from the addressable gem
-  module Uri
-    ALPHA      = "a-zA-Z"
-    DIGIT      = "0-9"
-    GEN_DELIMS = "\\:\\/\\?\\#\\[\\]\\@"
-    SUB_DELIMS = "\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\="
-    RESERVED   = GEN_DELIMS + SUB_DELIMS
-    UNRESERVED = ALPHA + DIGIT + "\\-\\.\\_\\~"
-    PCHAR      = UNRESERVED + SUB_DELIMS + "\\:\\@"
-    SCHEME     = ALPHA + DIGIT + "\\-\\+\\."
-    AUTHORITY  = PCHAR
-    PATH       = PCHAR + "\\/"
-    QUERY      = PCHAR + "\\/\\?"
-    FRAGMENT   = PCHAR + "\\/\\?"
-    #
-    PATHSEG    = ""
-  end
-end
-#
-#
+# Notes:
 #
+# * disambiguation: `generic disambiguates specifics` -- `["Alien", "Alien_(law)"]` and `["Alien", "Alien_(film)"]`
+# * redirects:      `dupe    redirects to  actual`    -- `["Oxygen-13", "Isotopes_of_oxygen"]`
+# * page_link:      `from    links to      into`      -- `["Achilles", "Greeks"]
+
 module Dbpedia
 
-  DBLQ_STRING_C   = '"(?<%s>\\\"|[^\"]+)+"'
   DECIMAL_NUM_RE  = '[\-\+\d]+\.\d+'
+  URI_PATHCHARS   = '\w\-\.\'~!$&()*+,;=:@'
+  # all backslash-escaped character, or non-quotes, up to first quote
+  DBLQ_STRING_C   = '"(?<%s>(?:\\.|[^\"])*)"'
+
+  # output flavors:
+  #
+  # :abstract_long   :abstract_short      :category :category_reln  :disambiguation
+  # :external_link   :geo_coordinates     :homepage :instance_of    :page_id :page_link
+  # :persondata_reln :property :redirects :sameas   :subject :title :wikipedia_link
+  #
+
+  MAPPING_INFO = {
+    # atomic topic properties
+    title:               { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,  :title,                                      ],  },
+    page_id:             { kind: :page_id,             fields: [:page_id, :wp_ns, :wikipedia_id, :wikipedia_pageid,                           ],  },
+    abstract_short:      { kind: :abstract_short,      fields: [:page_id, :wp_ns, :wikipedia_id, :abstract,                                   ],  },
+    abstract_long:       { kind: :abstract_long,       fields: [:page_id, :wp_ns, :wikipedia_id, :abstract,                                   ],  },
+    wikipedia_lang:      { kind: :skip,                fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :url,     :slug,  :lang,                  ],  },
+    wikipedia_link:      { kind: :wikipedia_link,      fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :url,     :slug,  :revision_id,           ],  },
+    wikipedia_backlink:  { kind: :skip,                fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :url,     :slug,  :revision_id,           ],  },
+    geo_coordinates:     { kind: :geo_coordinates,     fields: [:page_id, :wp_ns, :wikipedia_id, :lat,        :lng,                           ],  },
+    geo_coord_skip_a:    { kind: :skip,                fields: [],      },
+    geo_coord_skip_b:    { kind: :skip,                fields: [],      },
+    # links between topics
+    page_link:           { kind: :page_link,           fields: [:page_id, :wp_ns, :from_id,      :relation,   :into_id,                       ],  },
+    disambiguation:      { kind: :disambiguation,      fields: [:page_id, :wp_ns, :generic_wpid, :relation,   :specific_wpid,                 ],  },
+    redirects:           { kind: :redirects,           fields: [:page_id, :wp_ns, :dupe_id,      :relation,   :wikipedia_id,                  ],  },
+    # external links and sameas'es
+    external_link:       { kind: :external_link,       fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :weblink_url,                   ],  },
+    homepage:            { kind: :homepage,            fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :weblink_url,                   ],  },
+    geonames:            { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :geonames_id,                   ],  },
+    musicbrainz:         { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :musicbrainz_type,  :musicbrainz_id,], },
+    nytimes:             { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :nytimes_id,                    ],  },
+    pnd:                 { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :pnd_id,                        ],  },
+    uscensus:            { kind: :sameas,              fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :country_id, :state_id, :kind, :adm2_id, :adm3_id, :adm4_id],  },
+    # category links
+    category_skos_type:  { kind: :instance_of,         fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :obj_class                      ],  },
+    category_skos_title: { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :val_type,   :category_title,   ],  },
+    category:            { kind: :category,            fields: [:page_id, :wp_ns, :wikipedia_id, :flavor,     :specific_wpid,                 ],  },
+    category_subject:    { kind: :subject,             fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :into_wpid,                     ],  },
+    category_reln:       { kind: :category_reln,       fields: [:page_id, :wp_ns, :wikipedia_id, :relation,   :into_wpid,                     ],  },
+    # properties
+    wordnet:             { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :wn_reln,    :wn_class,   :wn_pos, :wn_idx,  ],  },
+    property_bool:       { kind: :property_bool,       fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_int:        { kind: :property_int,        fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_float:      { kind: :property_float,      fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_date:       { kind: :property_date,       fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_yearmonth:  { kind: :property_yearmonth,  fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_monthday:   { kind: :property_monthday,   fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_str:        { kind: :property_str,        fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    #
+    persondata_reln:     { kind: :persondata_reln,     fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :into_wpid,                     ],  },
+    # persondata_type:   { kind: :# persondata_type,   fields: [:page_id, :wp_ns, :wikipedia_id, :property,                                   ],  },
+    property_foaf:       { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :val,              ],  },
+    property_desc:       { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :val_type,   :name,             ],  },
+    yago:                { kind: :category,            fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :obj_class,                     ],  },
+    instance_type_a:     { kind: :instance_of,         fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :obj_class,                     ],  },
+    instance_type_b:     { kind: :instance_of,         fields: [:page_id, :wp_ns, :wikipedia_id, :scheme,     :obj_class,                     ],  },
+    property_specmap:    { kind: :property,            fields: [:page_id, :wp_ns, :wikipedia_id, :property,   :units,        :val,            ],  },
+    # topical_concepts:  { kind: :# topical_concepts,  fields: [:page_id, :wp_ns, :wikipedia_id, :skos_subject   :x,                          ],  },
+  }
 
   RDF_RES = {
+    # type descriptions
+    dbpedia_class:    'http://dbpedia\.org/class/(?<%s>[^>\s]+)',
     dbpedia_ontb:     'http://dbpedia\.org/ontology',
     dbpedia_ont:      'http://dbpedia\.org/ontology/(?<%s>[\w\/]+)',
     dbpedia_prop:     'http://dbpedia\.org/property/(?<%s>\w+)',
-    dbpedia_class:    'http://dbpedia\.org/class/(?<%s>[^>\s]+)',
-    dbpedia_rsrc:     "http://dbpedia\\.org/resource/(?<%s>[#{Re::Uri::PCHAR}%%\/]+)",
-    wikipedia_rsrc:   "http://\\w\\w\\.wikipedia\\.org/wiki/(?<%s>[#{Re::Uri::PCHAR}%%\/]+)",
-    dbpedia_value:    '"(?<%s>(?:\\\"|[^\"]+)*)"\\^\\^<http://dbpedia\.org/datatype/(?<%s>[a-zA-Z]+)>',
-    foaf_homepage:    'http://xmlns\.com/foaf/0\.1/homepage',
-    foaf_name:        'http://xmlns\.com/foaf/0\.1/name',
-    foaf_topic:       'http://xmlns\.com/foaf/0\.1/(?:isPrimaryTopicOf|primaryTopic)',
-    foaf_prop:        'http://xmlns\.com/foaf/0\.1/(?<property>\w+)',
-    geonames_rsrc:    'http://sws\.geonames\.org/(?<%s>\d+)/',
-    georss_latlng:    '\"(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\\s(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\"@en',
-    georss_type:      'http://www\.georss\.org/georss/point',
-    musicbrainz_rsrc: 'http://zitgist\.com/music/(?<%s>\w+)/(?<%s>[a-f0-9\-]+)',
-    nytimes_rsrc:     'http://data\.nytimes\.com/(?<%s>[A-Z0-9]+)',
-    purl_subject:     'http://purl\.org/dc/terms/subject',
+    dbpedia_rsrc:     'http://dbpedia\.org/resource/(?<%s>['           + URI_PATHCHARS + '%%\/]+)',
+    yago_class:       'http://dbpedia\.org/class/(?<%s>yago)/(?<%s>['  + URI_PATHCHARS + '%%\/]+)',
+    wikipedia_rsrc:   '(?<%s>http://\w\w\.wikipedia\.org/wiki/(?<%s>[' + URI_PATHCHARS + '%%\/]+))',
+    wiki_category:    'http://en\.wikipedia\.org/wiki/Category:Futurama?oldid=485425712\\#absolute-line=1',
+    wiki_link_id:     'http://en\.wikipedia\.org/wiki/(?<%s>[^\?]+)\?oldid=(?<%s>\d+)(?:\\#absolute-line=(?<%s>\d+))?',
+    wiki_link_id_sec: 'http://en\.wikipedia\.org/wiki/(?<%s>[^\?]+)\?oldid=(?<%s>\d+)\\#?(?:section=(?<%s>.*?)\&relative-line=(?<%s>\d+))?(?:&?absolute-line=(?<%s>\d+))?',
     purl_desc:        'http://purl\.org/dc/elements/1\.1/(?<%s>description)',
     purl_lang:        'http://purl\.org/dc/elements/1\.1/language',
+    purl_subject:     'http://purl\.org/dc/terms/subject',
+    rdf_type:         'http://www\.w3\.org/1999/02/22-rdf-syntax-ns\\#type',
     rdf_comment:      'http://www\.w3\.org/2000/01/rdf-schema\\#comment',
-    rdf_eol:          '\\.',
-    rdf_float:        '\"(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\"\^\^<http://www\.w3\.org/2001/XMLSchema\#float>',
-    rdf_integer:      '\"(?<%s>\d+)\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#integer>',
-    rdf_date:         '\"(?<%s>\d\d\d\d-\d\d-\d\d)\"\^\^<http://www\.w3\.org/2001/XMLSchema\\#date>',
-    rdf_string:       '\"(?<%s>(?:\\\"|[^\"]+)*)"@(?<%s>\w+)\b',
     rdf_label:        'http://www\.w3\.org/2000/01/rdf-schema\\#label',
-    rdf_type:         'http://www\.w3\.org/1999/02/22-rdf-syntax-ns\\#type',
+    # external links and sameas'es
     same_as:          'http://www\.w3\.org/2002/07/owl\\#sameAs',
-    skos_concept:     'http://www\.w3\.org/2004/02/skos/core\\#(?<%s>[a-zA-Z]+)',
-    skos_subject:     'http://www\.w3\.org/2004/02/skos/core\\#subject',
+    wordnet_inst:     'http://www\.w3\.org/2006/03/wn/wn20/instances/(?<%s>synset)-(?<%s>\w+)-(?<%s>noun)-(?<%s>[0-9]+)',
+    musicbrainz_rsrc: 'http://zitgist\.com/music/(?<%s>\w+)/(?<%s>[a-f0-9\-]+)',
+    nytimes_rsrc:     'http://data\.nytimes\.com/(?<%s>[A-Z0-9]+)',
+    geonames_rsrc:    'http://sws\.geonames\.org/(?<%s>\d+)/',
+    georss_type:      'http://www\.georss\.org/georss/point',
     wgs_latorlng:     'http://www\.w3\.org/2003/01/geo/wgs84_pos\\#(?:lat|long)',
+    #                  http://www.rdfabout.com/rdf/usgov/geo/  us     /     ak       /    counties   /bethel_area  /an_subarea   /aniak          >
+    uscensus_url:     'http://www.rdfabout.com/rdf/usgov/geo/(?<%s>us)/(?<%s>\w\w)(?:/(?<%s>counties)/(?<%s>\w+)(?:/(?<%s>\w+)\/?(?<%s>\w+)?)?)?',
+    # category links
+    skos_subject:     'http://www\.w3\.org/2004/02/skos/core\\#subject',
+    skos_concept:     'http://www\.w3\.org/2004/02/skos/core\\#(?<%s>[a-zA-Z]+)',
+    foaf_homepage:    'http://xmlns\.com/foaf/0\.1/homepage',
+    foaf_name:        'http://xmlns\.com/foaf/0\.1/name',
+    foaf_topic:       'http://xmlns\.com/foaf/0\.1/(?:isPrimaryTopicOf|primaryTopic)',
+    foaf_prop:        'http://xmlns\.com/foaf/0\.1/(?<property>\w+)',
+    # property values
+    georss_latlng:    '\"(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\\s(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\"@\w\w',
+    rdf_eol:          '\\.',
+    #
+    rdf_bool:         '\"(?<%s>true|false                        )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>boolean)>',
+    rdf_date:         '\"(?<%s>-?\d\d\d\d-\d\d-\d\d              )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>date)>',
+    rdf_yearmonth:    '\"(?<%s>-?\d\d\d\d-\d\d                   )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>gYearMonth)>',
+    rdf_monthday:     '\"(?<%s>--\d\d-\d\d                       )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>gMonthDay)>',
+    rdf_int:          '\"(?<%s>[\+\-]?\d+                        )\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>integer|gYear|positiveInteger|nonNegativeInteger)>',
+    rdf_float:        '\"(?<%s>[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?)\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>float|double)>',
+    # all backslash-escaped character, or non-quotes, up to first quote
+    rdf_string:       '"(?<%s>(?:\\\\.|[^\"])*)"@en',
+    dbpedia_value:    '"(?<%s>(?:\\\\.|[^\"])*)"\\^\\^<http://dbpedia\.org/datatype/(?<%s>[a-zA-Z]+)>',
+    #
     url_loose:        '(?<%s>(?:https?|ftp)://(?:[a-zA-Z0-9\-]+\.)+(?:[a-zA-Z\-]+)[^\s>]*)',
-    wiki_category:    'http://en\.wikipedia\.org/wiki/Category:Futurama?oldid=485425712\\#absolute-line=1',
-    wiki_link_id:     'http://en\.wikipedia\.org/wiki/(?<%s>[^\?]+)\?oldid=(?<%s>\d+)\\#?(?:absolute-line=(?<%s>\d+))?',
-    wiki_link_id_sec: 'http://en\.wikipedia\.org/wiki/(?<%s>[^\?]+)\?oldid=(?<%s>\d+)\\#?(?:section=(?<%s>.*?)\&relative-line=(?<%s>\d+))?(?:&?absolute-line=(?<%s>\d+))?',
-    wordnet_inst:     'http://www\.w3\.org/2006/03/wn/wn20/instances/synset-(?<%s>\w+)-noun-(?<%s>[0-9]+)',
-    yago_class:       'http://dbpedia\.org/class/yago',
-    rdf_value:        '\"(?<%s>
-      -?\d\d\d\d-\d\d-\d\d|-?\d\d\d\d-\d\d|--\d\d-\d\d|
-      [\+\-]?\d+  |
-      [\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?|
-      true|false)\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>integer|date|gYearMonth|gMonthDay|gYear|positiveInteger|nonNegativeInteger|float|double|boolean)>',
-
-    schema_type:      'http://(?<%s>www\\.w3\\.org/2002/07/owl|schema\\.org|dbpedia\\.org/ontology|purl\\.org/ontology|xmlns.com/foaf/0\\.1)[/\#]([^>]+)'
+    # rdf_value:        '\"(?<%s>-?\d\d\d\d-\d\d-\d\d|-?\d\d\d\d-\d\d|--\d\d-\d\d|[\+\-]?\d+|[\+\-]?\d+\.\d+(?:[eE][\+\-]?\d+)?|true|false)\"\\^\\^<http://www\.w3\.org/2001/XMLSchema\\#(?<%s>integer|date|gYearMonth|gMonthDay|gYear|positiveInteger|nonNegativeInteger|float|double|boolean)>',
+    schema_type:      'http://(?<%s>www\\.w3\\.org/2002/07/owl|schema\\.org|dbpedia\\.org/ontology|purl\\.org/ontology|xmlns.com/foaf/0\\.1)[/\\#](?<%s>[^>]+)'
   }
 
-  idx = 0;
-  RDF_RES.each{|flavor, re_str| Regexp.new(re_str.gsub('%s'){|x| "l#{idx+=1}" }) }
+  SCHEMA_SCHEMES = {
+    'www.w3.org/2002/07/owl' => 'owl',
+    'schema.org'             => 'schemaorg',
+    'dbpedia.org/ontology'   => 'dbpedia',
+    'purl.org/ontology'      => 'purl',
+    'xmlns.com/foaf/0.1'     => 'foaf'
+  }
 
+  # lookup regexp in above table, sub in variable names
   private
   def self.r(regexp_name, *args)
     RDF_RES[regexp_name] % args
   end
   public
 
-  MAPPINGS = {
-
-    geo_coordinates:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:georss_type)}>                        \s#{r(:georss_latlng, :lat, :lng)}               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    geo_coord_skip_a:      %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<http://www\.opengis\.net/gml/_Feature>        \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    geo_coord_skip_b:      %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:wgs_latorlng)}>                       \s#{r(:rdf_float, :val)}                         \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    wordnet:               %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_prop, :property)}>            \s<#{r(:wordnet_inst, :wn_class, :wn_idx)}>      \s#{r(:rdf_eol)}  \z}x,
-    geonames:              %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:geonames_rsrc, :geonames_id)}>           \s#{r(:rdf_eol)}  \z}x,
-
-    pnd:                   %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/individualisedPnd>     \s#{r(:rdf_string, :name, :nlang)}               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    disambiguations:       %r{\A<#{r(:dbpedia_rsrc, :generic_wikipedia_id)}> \s<#{r(:dbpedia_ontb)}/wikiPageDisambiguates> \s<#{r(:dbpedia_rsrc, :specific_wikipedia_id)}>  \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
-    page_ids:              %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/wikiPageID>            \s#{r(:rdf_integer,   :wikipedia_pageid)}        \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}>     \s#{r(:rdf_eol)}  \z}x,
-    redirects:             %r{\A<#{r(:dbpedia_rsrc, :dupe_of)}>              \s<#{r(:dbpedia_ontb)}/wikiPageRedirects>     \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>           \s#{r(:rdf_eol)}   \z}x,
-
-    article_categories:    %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:purl_subject)}>                       \s<#{r(:dbpedia_rsrc, :specific_wikipedia_id)}>  \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
-    categories_skos:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:skos_concept, :skos_relation)}>          \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
-    categories_skos_label: %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:skos_concept, :skos_relation)}>       \s#{r(:rdf_string, :val, :val_lang)}             \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
-    categories_skos_reln:  %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:skos_concept, :skos_relation)}>       \s<#{r(:dbpedia_rsrc, :category_b)}>             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    category_labels:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_label)}>                          \s#{r(:rdf_string,:category_wikipedia_id, :ctl)} \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
-
-    abstracts_short:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_comment)}>                        \s#{r(:rdf_string,:abstract, :al)}               \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
-    abstracts_long:        %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/abstract>              \s#{r(:rdf_string,:abstract, :al)}               \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}xm,
-    titles:                %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_label)}>                          \s#{r(:rdf_string,:wikipedia_id2,  :tl )}        \s<#{r(:wiki_link_id, :wikipedia_id2, :wikipedia_pageid, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
-    nytimes:               %r{\A<#{r(:nytimes_rsrc, :nyt_id)}>               \s<#{r(:same_as)}>                            \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>           \s#{r(:rdf_eol)}   \z}x,
-
-    external_links:        %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/wikiPageExternalLink>  \s<#{r(:url_loose, :weblink_url)}>               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    homepages:             %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:foaf_homepage)}>                      \s<#{r(:url_loose, :weblink_url)}>               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    wikipedia_links:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:foaf_topic)}>                         \s<#{r(:url_loose, :weblink_url)}>               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    wikipedia_lang:        %r{\A<#{r(:wikipedia_rsrc, :wikipedia_id)}>       \s<#{r(:purl_lang)}>                          \s#{r(:rdf_string, :lang, :wll_lang)}            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    wikipedia_rev:         %r{\A<#{r(:wikipedia_rsrc, :wikipedia_id)}>       \s<#{r(:foaf_topic)}>                         \s<#{r(:dbpedia_rsrc, :wikipedia_id2)}>          \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    musicbrainz:           %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:musicbrainz_rsrc, :musicbrainz_type, :musicbrainz_id)}>    \s#{r(:rdf_eol)}  \z}x,
-
-    properties_map_val:    %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_value,  :val, :val_type)}             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    properties_map_str:    %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_string, :val, :val_lang)}             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    properties_map_foaf:   %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:foaf_prop, :property)}>               \s#{r(:rdf_string, :val, :val_lang)}             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    properties_specmap:    %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:dbpedia_value, :val, :dbpedia_units)}     \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-
-    yago:                  %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:schema_type, :schema, :schema_type)}>    \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-
-    persondata_type:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<http://xmlns.com/foaf/0\.1/Person>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    persondata_reln:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s<#{r(:dbpedia_rsrc, :target_wikipedia_id)}>    \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    persondata_prop:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_value, :val, :val_type)}              \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    persondata_foaf:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:foaf_prop, :property)}>               \s#{r(:rdf_string,:name, :name_lang)}            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-    persondata_desc:       %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:purl_desc, :property)}>               \s#{r(:rdf_string,:name, :name_lang)}            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}>  \s#{r(:rdf_eol)}  \z}x,
-
-
-    #topical_concepts:      %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>       \s<#{r(:skos_subject)}>                       \s<#{r(:x, )} \z}x,  #<http://dbpedia.org/resource/Futurama> \s<#{r(:wiki_category)}> .
-    # uscensus:               %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>        \s<#{r(:same_as)}>                            \s<#{r(:url_loose, :census_url)}>   \s#{r(:rdf_eol)}  \z}x,
-    # instance_types:        %r{\A<#{r(:dbpedia_rsrc, :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:dbpedia_ont, :specific_wikipedia_id)}> }x,#        \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :wikipedia_pageid, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+  MAPPING_RES = {
+    # atomic topic properties
+    title:               %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_label)}>                          \s#{r(:rdf_string, :title )}                   \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}x,
+    page_id:             %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/wikiPageID>            \s#{r(:rdf_int,    :wikipedia_pageid, :_dtyp)} \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_lang:      %r{\A<#{r(:wikipedia_rsrc,  :url, :slug)}>         \s<#{r(:purl_lang)}>                          \s#{r(:rdf_string, :lang)}                       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_link:      %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:foaf_topic)}>                         \s<#{r(:wikipedia_rsrc, :url, :slug)}>         \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    wikipedia_backlink:  %r{\A<#{r(:wikipedia_rsrc,  :url, :slug)}>        \s<#{r(:foaf_topic)}>                         \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    abstract_short:      %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_comment)}>                        \s#{r(:rdf_string, :abstract)}                 \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}x,
+    abstract_long:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/abstract>              \s#{r(:rdf_string, :abstract)}                 \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}xm,
+    geo_coordinates:     %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:georss_type)}>                        \s#{r(:georss_latlng, :lat, :lng)}             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    geo_coord_skip_a:    %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<http://www\.opengis\.net/gml/_Feature>      \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    geo_coord_skip_b:    %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:wgs_latorlng)}>                       \s#{r(:rdf_float, :val, :_dtyp)}               \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    # links between topic
+    page_link:           %r{\A<#{r(:dbpedia_rsrc,    :from_id)}>              \s<#{r(:dbpedia_ontb)}/wikiPageWikiLink>      \s<#{r(:dbpedia_rsrc, :into_id)}>              \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    disambiguation:      %r{\A<#{r(:dbpedia_rsrc,    :generic_wpid)}>         \s<#{r(:dbpedia_ontb)}/wikiPageDisambiguates> \s<#{r(:dbpedia_rsrc, :specific_wpid)}>        \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    redirects:           %r{\A<#{r(:dbpedia_rsrc,    :dupe_id)}>              \s<#{r(:dbpedia_ontb)}/wikiPageRedirects>     \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>                                                                                                                       \s#{r(:rdf_eol)}   \z}x,
+    # external links and sameas'es
+    external_link:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/wikiPageExternalLink>  \s<#{r(:url_loose, :weblink_url)}>             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    homepage:            %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:foaf_homepage)}>                      \s<#{r(:url_loose, :weblink_url)}>             \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    geonames:            %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:geonames_rsrc, :geonames_id)}>                                                                                                                       \s#{r(:rdf_eol)}  \z}x,
+    musicbrainz:         %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:musicbrainz_rsrc, :musicbrainz_type, :musicbrainz_id)}>                                                                                              \s#{r(:rdf_eol)}  \z}x,
+    nytimes:             %r{\A<#{r(:nytimes_rsrc,    :nytimes_id)}>           \s<#{r(:same_as)}>                            \s<#{r(:dbpedia_rsrc, :wikipedia_id)}>                                                                                                                       \s#{r(:rdf_eol)}   \z}x,
+    uscensus:            %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:same_as)}>                            \s<#{r(:uscensus_url, :country_id, :state_id, :kind, :adm2_id, :adm3_id, :adm4_id)}>                                                                         \s#{r(:rdf_eol)}  \z}x,
+    pnd:                 %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ontb)}/individualisedPnd>     \s#{r(:rdf_string, :pnd_id)}                   \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    # category links
+    category:            %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:purl_subject)}>                       \s<#{r(:dbpedia_rsrc, :specific_wpid)}>        \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    category_skos_type:  %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:skos_concept, :obj_class)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    category_subject:    %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:skos_subject, :relation)}>            \s<#{r(:dbpedia_rsrc, :into_wpid)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    category_reln:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:skos_concept, :relation)}>            \s<#{r(:dbpedia_rsrc, :into_wpid)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    category_skos_title: %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:skos_concept, :relation)}>            \s#{r(:rdf_string, :category_title)}           \s<#{r(:wiki_link_id, :wikipedia_id2, :revision_id, :article_lineno)}>                                        \s#{r(:rdf_eol)}  \z}x,
+    # properties
+    wordnet:             %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_prop, :property)}>            \s<#{r(:wordnet_inst, :wn_reln, :wn_class, :wn_pos, :wn_idx)}>                                                                                               \s#{r(:rdf_eol)}  \z}x,
+    #
+    property_bool:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_bool,      :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_int:        %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_int,       :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_float:      %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_float,     :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_date:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_date,      :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_yearmonth:  %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_yearmonth, :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_monthday:   %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_monthday,  :val, :val_type) }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_str:        %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:rdf_string,    :val)            }       \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    #
+    persondata_reln:     %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s<#{r(:dbpedia_rsrc, :into_wpid)}>            \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_foaf:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:foaf_prop, :property)}>               \s#{r(:rdf_string, :val)}                      \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_desc:       %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:purl_desc, :property)}>               \s#{r(:rdf_string,:name)}                      \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    yago:                %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:yago_class,  :scheme, :obj_class)}>                                                                                                                  \s#{r(:rdf_eol)}  \z}x,
+    instance_type_a:     %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:dbpedia_ont,          :obj_class)}>    \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    instance_type_b:     %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:rdf_type)}>                           \s<#{r(:schema_type, :org,    :obj_class)}>    \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    property_specmap:    %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:dbpedia_ont, :property)}>             \s#{r(:dbpedia_value, :val, :units)}           \s<#{r(:wiki_link_id_sec, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno)}> \s#{r(:rdf_eol)}  \z}x,
+    # topical_concepts:  %r{\A<#{r(:dbpedia_rsrc,    :wikipedia_id)}>         \s<#{r(:skos_subject)}>                       \s<#{r(:x, )}> \z},
   }
-
-  # ap MAPPINGS
+  MAPPING_RES.each{|re_name, re| MAPPING_INFO[re_name][:re] = re }
+  SKIPPAPLE_FIELDS = [:flavor, :wikipedia_id2, :revision_id, :article_section, :section_lineno, :article_lineno, :val_lang, :name_lang, :_dtyp]
 
   class RdfExtractor < Wukong::Streamer::LineStreamer
     include MungingUtils
+    attr_accessor :flavor, :kind, :filename, :regexps, :seen_keys, :seen_props
+
+    def initialize(*args)
+      @seen_keys  = Hash.new(0)
+      @seen_props = Hash.new(0)
+    end
+
+    def record_for_flavor(kind, fields, flavor, hsh)
+      hsh.merge!( wp_ns: 0, flavor: flavor )
+      return if kind == :skip
+
+      case flavor
+      when :property_str, :property_foaf   then  hsh[:val] = MultiJson.encode(hsh[:val])
+      when :abstract_long, :abstract_short then  hsh[:abstract] = MultiJson.encode(hsh[:abstract])
+      when :title                          then  hsh[:title] = MultiJson.encode(hsh[:title]) ; hsh[:property] = 'title'
+      when :category_skos_title            then  hsh[:category_title] = MultiJson.encode(hsh[:category_title])
+      when :category_skos_type then hsh[:scheme] = 'skos'
+      when :category_subject   then hsh[:scheme] = 'subject'
+      when :instance_type_a    then hsh[:scheme] = 'dbpedia'
+      when :instance_type_b
+        hsh[:scheme] = SCHEMA_SCHEMES[hsh.delete(:org)]
+        return if hsh[:scheme] == 'owl'
+      when :wikipedia_link, :wikipedia_backlink
+        raise "Titles disagree!" unless hsh[:slug] == hsh[:wikipedia_id]
+      end
 
-    def emit(hsh)
-      super [MultiJson.encode(hsh)]
+      # record seen properties, seen fields
+      hsh.except(*fields).except(*SKIPPAPLE_FIELDS).
+        each{|key, val| @seen_keys[key] += 1 if val.present? }
+      seen_props[hsh[:property]] += 1 if hsh[:property].present?
+      sanity_check(hsh)
+      #
+      [kind] + hsh.values_at(*fields)
+    end
+
+    def sanity_check(hsh)
+      hsh.each{|key,val| raise if CONTROL_CHARS_RE =~ val.to_s }
+    end
+
+    def after_stream
+      Log.info ["seen keys:", seen_keys.inspect, "seen props:", seen_props.inspect].join("\t")
     end
 
     def process(line)
       return if line =~ /\A(?:\#|$)/
-      if line =~ /=> \w+\.\w+ <=/ then yield [line] ; return ; end
-      MAPPINGS.each do |flavor, re|
-        next unless mm = re.match(line)
-        yield( { flavor: flavor }.merge(mm.as_hash) )
+      if (line =~ /=> \w+\.\w+ <=/) then yield [line] ; return ; end
+
+      MAPPING_INFO.each do |flavor, info|
+        next unless mm = info[:re].match(line)
+        yield record_for_flavor(info[:kind], info[:fields], flavor, mm.as_hash)
         return
       end
-      puts [line]
+
+      Log.warn ['not found:', line].join("\t")
     end
   end
 end
 
 
-# META = {
-#   geo_coordinates:    [:field,      'geo_coordinates_en.nq',                      ],
-#   wordnet:            [:joinkey,    'wordnet_links.nt',                           ],
-#   geonames:           [:joinkey,    'geonames_links.nt',                          ],
-#   properties_specmap: [:properties, 'specific_mappingbased_properties_en.nq',     ],
-#   properties_mapped:  [:properties, 'mappingbased_properties_unredirected_en.nq', ],
-#   pnd:                [:joinkey,    'pnd_en.nq',                                  ],
-#   disambiguations:    [:pagelink,   'disambiguations_unredirected_en.nq',         ],
-#   external_links:     [:weblink,    'external_links_en.nq',                       ],
-#   page_ids:           [:field,      'page_ids_en.nq',                             ],
-#   redirects:          [:pagelink,   'redirects_transitive_en.nt',                 ],
-#   article_categories: [:categories, 'article_categories_en.nq',                   ],
-#   instance_types:     [:categories, 'instance_types_en.nq',                       ],
-#   categories_skos:    [:meta,       'skos_categories_en.nq',                      ],
-#   abstracts_long:     [:field,      'long_abstracts_en.nq',                       ],
-#   abstracts_short:    [:field,      'short_abstracts_en.nq',                      ],
-#   category_labels:    [:meta,       'category_labels_en.nq',                      ],
-#   titles:             [:field,      'labels_en.nq',                               ],
-#   musicbrainz:        [:joinkey,    'musicbrainz_links.nt',                       ],
-#   nytimes:            [:joinkey,    'nytimes_links.nt',                           ],
-#   uscensus:           [:joinkey,    'uscensus_links.nt',                          ],
-#   topical_concepts:   [             'topical_concepts_unredirected_en.nq',        ],
-#   homepages:          [:weblink,    'homepages_en.nq',                            ],
-#   wikipedia_links:    [:field,      'wikipedia_links_en.nq',                      ],
-#   persondata:         [:properties, 'persondata_unredirected_en.nq',              ],
-#   yago:               [:joinkey,    'yago_links.nt',                              ],
-# }
-
 
 Wukong::Script.new(Dbpedia::RdfExtractor, nil).run