ul-wukong 4.1.0

data/examples/munging/geo/geo_json.rb

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+module Wukong
+  module Data
+    class GeoJson           ; include Gorillib::Model ; end
+    class GeoJson::Geometry ; include Gorillib::Model ; end
+
+    class GeoJson
+      include Gorillib::Model::LoadFromJson
+      include Gorillib::Model::Indexable
+      field :type,       String
+      field :id,         String
+      field :geometry,   GeoJson::Geometry
+      field :properties, GenericModel
+
+      def self.load(*args)
+        load_json(*args) do |val|
+          p val.properties
+          p val.properties.to_place
+        end
+      end
+
+    end
+
+    class GeoJson::Geometry
+      field :type,        String
+      field :coordinates, Array
+
+      def point?
+        type == 'Point'
+      end
+
+      def longitude
+        return nil if coordinates.blank?
+        raise "Longitude only available for Point objects" unless point?
+        coordinates[0]
+      end
+      def latitude
+        return nil if coordinates.blank?
+        raise "Latitude only available for Point objects" unless point?
+        coordinates[1]
+      end
+    end
+
+    class GeonamesGeoJson < GeoJson
+      def receive_properties(hsh)
+        if hsh.respond_to?(:merge)
+          super(hsh.merge(geo_json_id: id, longitude: geometry.longitude, latitude: geometry.latitude))
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/examples/munging/geo/geo_models.rb

@@ -0,0 +1,69 @@
+module Geo
+
+  class Place
+    include Gorillib::Model
+    include Gorillib::Model::Indexable
+
+    field :geonames_id,       String
+    field :country_id,        String,  doc: "ISO 3166 2-letter alphanumeric id ('us', 'mx', etc). Must be lowercase"
+    field :admin1_id,         String
+    field :feature_cat,       String
+    field :feature_subcat,    String
+    #
+    field :name,              String
+    #
+    field :timezone,          String
+    field :elevation,         Float
+    field :longitude,         Float
+    field :latitude,          Float
+    #
+    field :alternate_names,   String, default: ""
+
+    def names
+      ([name] + alternate_names.split("|")).compact_blank
+    end
+
+    def coordinates
+      { longitude: longitude, latitude: latitude, elevation: elevation }.compact
+    end
+
+    def self.slugify_name(val)
+      val.downcase.
+        gsub(/(?:\s+and\s+|\s+-\s+|[^[:alpha:]\-]+)/, '-').
+        gsub(/\A-*(.+?)-*\z/, '\1')
+    end
+  end
+
+  class AdministrativeArea < Place
+    field :population,        Integer
+    field :official_name,     String
+    def names ; super.tap{|arr| arr.insert(1, official_name) }.uniq.compact_blank ; end
+  end
+
+  class Country < AdministrativeArea
+    field :country_al3id,   String, identifier: true,  doc: "ISO 3166 3-letter alphanumeric id ('usa', 'mex', etc). Must be lowercase."
+    field :country_numid,   Integer, identifier: true, doc: "ISO 3166 numeric identifier ('usa' = 840)"
+    field :tld_id,          String, doc: "TLD (top-level domain) identifier"
+  end
+
+  class CountryNameLookup
+    include Gorillib::Model
+    include Gorillib::Model::Indexable
+    include Gorillib::Model::LoadFromTsv
+    index_on :slug
+
+    field :country_id, String
+    field :country_al3id, String
+    field :country_numid, Integer
+    field :tld_id,        String
+    field :geonames_id,   String
+    field :name,          String
+    field :slug,          String
+    field :alt_name,      String
+
+    def self.load(filename=nil)
+      filename ||= :country_name_lookup
+      @values = load_tsv(filename)
+    end
+  end
+end

data/examples/munging/geo/geonames_models.rb

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+# {"type":"Feature",
+#  "id":"3cc54602f2d69c1111dc35f0aaa92240",
+#  "geometry":{"type":"Point","coordinates":[42.5,11.5]},
+#  "properties":{
+#    "geonameid":"223816","country_code":"DJ","admin1_code":"00",
+#    "feature_code":"PCLI","feature_class":"A",
+#    "asciiname":"Republic of Djibouti","name":"Republic of Djibouti","alternatenames":"Cîbûtî,...",
+#    "modification_date":"2011-07-09",
+#    "timezone":"Africa/Djibouti","gtopo30":"668","population":"740528"}}
+
+
+# {"type":"Feature","id":"5b66ac7270763facfe1e9ab9c1bf99f8",
+# "geometry":{"type":"Point","coordinates":[-98.5,39.76]},
+# "properties":{
+# "modification_date":"2011-04-27","_type":"geo/geonames_country",
+# "asciiname":"United States","name":"United States","gtopo30":"537","geonameid":"6252001",
+# "feature_code":"PCLI","country_code":"US","feature_class":"A",
+# "alternatenames":"...","admin1_code":"00","population":"310232863"}}
+
+module Geo
+
+  class GeonamesPlace
+    include Gorillib::Model
+    class_attribute :place_klass ; self.place_klass = ::Geo::Place
+
+    field :name,              String
+    field :asciiname,         String
+    field :geonameid,         String
+    field :country_code,      String
+    field :admin1_code,       String, blankish: [0, "0", "00", nil, ""]
+    field :feature_code,      String
+    field :feature_class,     String
+    #
+    field :modification_date, String
+    field :timezone,          String
+    #
+    field :gtopo30,           Float,   blankish: ["-9999", -9999, nil, ""], doc: "Elevation in the [GTOPO30](http://en.wikipedia.org/wiki/GTOPO30) model"
+    field :longitude,         Float
+    field :latitude,          Float
+    #
+    field :population,        Integer, blankish: [0, "0", nil, ""]
+    field :alternatenames,    String
+
+    # because 'Saint Helena, Ascension and Tristan da Cunha' is an official
+    # country name (and others like it
+    def alternate_names_with_pipes
+      # comma ',' with no spaces separates names; comma space ', ' is internal.
+      an = alternatenames.gsub(/,/, '|').gsub(/\| /, ', ')
+      ([name, asciiname] + an.split('|')).uniq.join("|")
+    end
+
+    def to_place
+      attrs = {
+        name:            asciiname,
+        official_name:   name,
+        geonames_id:     "gn:#{geonameid}",
+        country_id:      country_code.downcase,
+        admin1_id:       admin1_code,
+        feature_cat:     feature_class,
+        feature_subcat:  feature_code,
+        alternate_names: alternate_names_with_pipes,
+        updated_at:      modification_date,
+        timezone:        timezone,
+        elevation:       gtopo30,
+        longitude:       longitude,
+        latitude:        latitude,
+        population:      population,
+      }
+      place_klass.receive(attrs)
+    end
+  end
+
+  # Stub class: Geonames JSON elements have :_type = geo/geonames_country
+  class GeonamesCountry < GeonamesPlace
+    self.place_klass = Geo::Country
+  end
+
+  # http://download.geonames.org/export/zip/
+  #
+  # country code      : iso country code, 2 characters
+  # postal code       : varchar(20)
+  # place name        : varchar(180)
+  # admin name1       : 1. order subdivision (state) varchar(100)
+  # admin code1       : 1. order subdivision (state) varchar(20)
+  # admin name2       : 2. order subdivision (county/province) varchar(100)
+  # admin code2       : 2. order subdivision (county/province) varchar(20)
+  # admin name3       : 3. order subdivision (community) varchar(100)
+  # admin code3       : 3. order subdivision (community) varchar(20)
+  # latitude          : estimated latitude (wgs84)
+  # longitude         : estimated longitude (wgs84)
+  # accuracy          : accuracy of lat/lng from 1=estimated to 6=centroid
+  class GeonamesPostal
+    field :country_id,   String, doc: "iso country code, 2 characters"
+    field :postal_id,    String, doc: "varchar(20)"
+    field :name,         String, doc: "varchar(180)"
+    field :admin1_name,  String, doc: "1. order subdivision (state) varchar(100)"
+    field :admin1_id,    String, doc: "1. order subdivision (state) varchar(20)"
+    field :admin2_name,  String, doc: "2. order subdivision (county/province) varchar(100)"
+    field :admin2_id,    String, doc: "2. order subdivision (county/province) varchar(20)"
+    field :admin3_name,  String, doc: "3. order subdivision (community) varchar(100)"
+    field :admin3_id,    String, doc: "3. order subdivision (community) varchar(20)"
+    field :latitude,     String, doc: "estimated latitude (wgs84)"
+    field :longitude,    String, doc: "estimated longitude (wgs84)"
+    field :accuracy,     String, doc: "accuracy of lat/lng from 1=estimated to 6=centroid"
+  end
+end

data/examples/munging/geo/iso_codes.rb

@@ -0,0 +1,172 @@
+require 'active_support/lazy_load_hooks'
+require 'active_support/i18n'
+require 'active_support/inflector/transliterate'
+
+module Wukong
+
+  module Data
+    
+    # These classes use data from the
+    # [isocodes](http://pkg-isocodes.alioth.debian.org/) debian project. That
+    # package provides lists of various ISO standards (e.g. country, language,
+    # language scripts, and currency names) in one place, rather than repeated in
+    # many programs throughout the system.
+    #
+    class IsoCode
+      include Gorillib::Model
+      include Gorillib::Model::LoadFromTsv
+      include Gorillib::Model::Indexable
+
+      class_attribute :handle,      instance_writer: false
+      def self.load(filename=nil)
+        filename ||= [:geo_data, 'iso_codes', "iso_3166.tsv"]
+        @values = load_tsv(filename, num_fields: 4..6)
+      end
+    end
+
+    #
+    # ISO 3166 Country code
+    #
+    # Lists the 2-letter country code and "short" country name. The official ISO
+    # 3166 maintenance agency is ISO. The gettext domain is
+    # "iso_3166". [origin](http://www.iso.org/iso/country_codes)
+    #
+    class CountryCode < IsoCode
+      include ActiveSupport::Inflector
+      
+      self.handle = :iso_3166
+      index_on :alpha_2_code, :alpha_3_code, :country_numid, :name, :common_name, :official_name
+      field :alpha_2_code,    String, identifier: true
+      field :alpha_3_code,    String, identifier: true
+      field :country_numid,   Integer, identifier: true
+      field :name,            String
+      field :official_name,   String, blankish: ["", nil]
+      field :common_name,     String, blankish: ["", nil]
+
+      def names
+        [common_name, name, official_name].compact_blank
+      end
+      def self.for_any_name(val)
+        for_name(val){ for_common_name(val){ for_official_name(val) } }
+      end
+
+      def to_place
+        attrs = {
+          name:            transliterate(names.first),
+          official_name:   names.last,
+          country_id:      alpha_2_code.downcase,
+          alternate_names: names.join('|'),
+          country_al3id:   alpha_3_code.downcase,
+          country_numid:   country_numid,
+        }
+        Geo::Country.receive(attrs.compact_blank)
+      end
+    end
+
+    class CountryCode < IsoCode
+      self.handle = :iso_3166_3
+      field :alpha_3_code,    String, identifier: true
+      field :alpha_4_code,    String, identifier: true
+      field :country_numid,   Integer, identifier: true
+      field :country_names,   String
+      field :comment,         String
+      field :date_withdrawn,  String
+    end
+
+    #
+    # ISO 3166-2 Country Subdivision (Admin 1: state, region, etc) Code
+    #
+    # The ISO 3166 standard includes a "Country Subdivision Code", giving a code
+    # for the names of the principal administrative subdivisions of the
+    # countries coded in ISO 3166. The official ISO 3166-2 maintenance agency is
+    # ISO. The gettext domain is "iso_3166_2".
+    # <http://www.iso.org/iso/country_codes/background_on_iso_3166/iso_3166-2.htm>
+    #
+    class RegionCode < IsoCode
+      self.handle = :iso_3166_2
+      field :region_code,     String, identifier: true
+      field :country_code,    String
+      field :parent_region,   String
+      field :region_kind,     String
+      field :name,            String
+      alias_method :state_code, :region_code
+    end
+
+    #
+    # ISO 639 Language Code
+    #
+    # This lists the 2-letter and 3-letter language codes and language
+    # names. The official ISO 639 maintenance agency is the Library of
+    # Congress. The gettext domain is "iso_639".
+    # [origin](http://www.loc.gov/standards/iso639-2/)
+    #
+    class BasicLanguageCode < IsoCode
+      self.handle = :iso_639
+      field :iso_639_1_code,  String, identifier: true
+      field :iso_639_2B_code, String, identifier: true
+      field :iso_639_2T_code, String, identifier: true
+      field :name,            String, identifier: true
+    end
+
+    # ISO 639-3
+    #
+    # This is a further development of ISO 639-2, see above. All codes of ISO
+    # 639-2 are included in ISO 639-3. ISO 639-3 attempts to provide as complete
+    # an enumeration of languages as possible, including living, extinct,
+    # ancient, and constructed languages, whether major or minor, written or
+    # unwritten. The gettext domain is "iso_639_3". The official ISO 639-3
+    # maintenance agency is SIL International.
+    # [origin](http://www.sil.org/iso639-3/)
+    #
+    class LanguageCode < BasicLanguageCode
+      self.handle = :iso_639_3
+      field :language_id,     String, identifier: true
+      field :part1_code,      String
+      field :part2_code,      String
+      field :scope,           String
+      field :status,          String
+      field :language_kind,   String
+      field :name,            String
+      field :inverted_name,   String
+      field :reference_name,  String
+    end
+
+    #
+    # ISO 15924 Language Scripts (alphabet) names
+    #
+    # This lists the language scripts names. The official ISO 15924 maintenance
+    # agency is the Unicode Consortium. The gettext domain is "iso_15924".
+    # [origin](http://unicode.org/iso15924/)
+    #
+    class LanguageScriptCode < IsoCode
+      self.handle = :iso_15924
+      field :alpha_4_code,    String, identifier: true
+      field :script_numid,    Integer, identifier: true
+      field :name,            String
+    end
+
+    #
+    # ISO 4217 Currency Code
+    #
+    # This lists the currency codes and names. The official ISO 4217 maintenance
+    # agency is the British Standards Institution. The gettext domain is
+    # "iso_4217".
+    # [origin](http://www.bsi-global.com/en/Standards-and-Publications/Industry-Sectors/Services/BSI-Currency-Code-Service/)
+    #
+    class CurrencyCode < IsoCode
+      self.handle = :iso_4217
+      field :currency_code,   String, identifier: true
+      field :currency_numid,  Integer, identifier: true
+      field :name,            String
+    end
+
+    #
+    # Historic Currency Code
+    #
+    class HistoricCurrencyCode < CurrencyCode
+      self.handle = :historic_iso_4217
+      field :date_withdrawn,  String
+    end
+
+  end
+end

data/examples/munging/geo/reconcile_countries.rb

@@ -0,0 +1,124 @@
+require 'gorillib/model/reconcilable'
+# require_relative('./geo_models')
+# require_relative('./geo_json')
+
+module Geo
+
+  Place.class_eval do
+    include Gorillib::Model::Reconcilable
+
+    def adopt_alternate_names(that_val, _)
+      return true if that_val.blank?
+      names = "#{alternate_names}|#{that_val}".split("|")
+      names.uniq!
+      names.delete(name)
+      write_attribute :alternate_names, names.compact_blank.join("|")
+      true
+    end
+
+    def conflicting_attribute!(attr, this_val, that_val)
+      case attr
+      when :name, :official_name then return :pass
+      end
+      super
+    end
+
+  end
+
+  Country.class_eval do
+    index_on :country_id
+    field :iso_3166_active, :boolean
+  end
+
+
+  class FullIso3166
+    include Gorillib::Model
+    include Gorillib::Model::Reconcilable
+    include Gorillib::Model::LoadFromTsv
+    self.tsv_options = self.tsv_options.merge(num_fields: 6..8, pop_headers: true)
+
+    field :country_id,      String
+    field :tld_id,          String
+    field :iso_3166_3,      String
+    field :name,            String
+    field :code_status,     String
+    field :iso_3166_active, :boolean, blankish: ['N', false, nil, '']
+    field :year_granted,    String
+    field :notes,           String
+
+    def active?
+      iso_3166_active == "Y"
+    end
+
+    def to_place
+      Geo::Country.receive({
+          country_id:      country_id,
+          name:            name,
+          tld_id:          tld_id,
+          iso_3166_active: iso_3166_active,
+        })
+    end
+  end
+
+end
+
+# cd    Congo (Kinshasa)
+# um    Baker Island
+# um    Howland Island
+# um    Jarvis Island
+# um    Johnston Atoll
+# um    Kingman Reef
+# um    Midway Islands
+# um    Navassa Island
+# um    Palmyra Atoll
+# um    Wake Island
+# mi    Midway Islands
+# na    Netherlands Antilles
+# gs    South Georgia and the Islands
+# sj    Svalbard
+# wk    Wake Island
+# ps    West Bank
+# ps    West Bank and the Gaza Strip
+# ps    Gaza Strip
+
+class CountryReconciler
+
+  def self.load_reconciled_countries
+
+    Geo::FullIso3166.load_tsv([:geo_data, 'iso_codes/full_iso_3166.tsv']) do |raw_country|
+      Geo::Country.values << raw_country.to_place
+    end
+
+    Wukong::Data::CountryCode.load
+    Wukong::Data::CountryCode.values.each do |raw_country|
+      iso_country = raw_country.to_place
+      country = Geo::Country.for_country_id(iso_country.country_id){ Geo::Country.new }
+      country.adopt(iso_country)
+    end
+
+    Wukong::Data::GeonamesGeoJson.load_json(:geonames_countries) do |raw_feature|
+      gn_country = raw_feature.properties.to_place
+      country = Geo::Country.for_country_id(gn_country.country_id){ Geo::Country.new }
+      country.adopt(gn_country)
+    end
+
+    Geo::Country.values.sort_by!(&:country_id)
+  end
+end
+
+
+
+# {
+#   :xx => { :name => 'Iran' },
+#   :xx => 'Tanzania, United Republic',
+#   :xx => 'Palestinian Territory, Occupied',
+#   :xx =>
+# }
+# :kp => "North Korea"  Korea, Democratic People's Republic
+# :kr => "South Korea"  Korea, Republic of
+# :bn => "Brunei"
+# :bq => "Caribbean Netherlands"
+# Lao People's Democratic Republic
+#
+# :va Holy See (Vatican City State)
+# :vi Virgin Islands, U.S.