worlddb-models 2.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a6f4e2c609b98e4e47364d364685e82cdb315ab7
+  data.tar.gz: 9329d22376b1a85b186780ff0db0884de86f85af
+SHA512:
+  metadata.gz: ac729c3828ed5e4ffd2c54cf3e700bb1f6f20e0be8fc670b2b7c524700e50ef2544aa0fd1838ca72a7d16581fbc0def39628afd9cdb02a6c1c27b92e0321b6bf
+  data.tar.gz: 3c8e225f39c7481fca92ec4395f2ef262c8c91bc7065e85f69e8662bbdc67c6d4883d4a1b5b3cf4866653106bc3182111666b62b66b2238da2c33a5bdc3d0a59

data/HISTORY.md

@@ -0,0 +1,4 @@
+### 0.0.1 / 2012-11-05
+
+* Everything is new. First release
+

data/Manifest.txt

@@ -0,0 +1,43 @@
+HISTORY.md
+Manifest.txt
+README.md
+Rakefile
+lib/worlddb/deleter.rb
+lib/worlddb/matcher.rb
+lib/worlddb/models.rb
+lib/worlddb/models/city.rb
+lib/worlddb/models/city_comp.rb
+lib/worlddb/models/continent.rb
+lib/worlddb/models/continent_comp.rb
+lib/worlddb/models/country.rb
+lib/worlddb/models/country_code.rb
+lib/worlddb/models/country_comp.rb
+lib/worlddb/models/forward.rb
+lib/worlddb/models/lang.rb
+lib/worlddb/models/lang_comp.rb
+lib/worlddb/models/name.rb
+lib/worlddb/models/place.rb
+lib/worlddb/models/region.rb
+lib/worlddb/models/region_comp.rb
+lib/worlddb/models/tagdb/tag.rb
+lib/worlddb/models/tagdb/tagging.rb
+lib/worlddb/models/usage.rb
+lib/worlddb/patterns.rb
+lib/worlddb/reader.rb
+lib/worlddb/reader_file.rb
+lib/worlddb/reader_zip.rb
+lib/worlddb/readers/city.rb
+lib/worlddb/readers/country.rb
+lib/worlddb/readers/lang.rb
+lib/worlddb/readers/region.rb
+lib/worlddb/readers/usage.rb
+lib/worlddb/schema.rb
+lib/worlddb/stats.rb
+lib/worlddb/version.rb
+test/helper.rb
+test/test_fixture_matchers.rb
+test/test_model_city.rb
+test/test_model_comp.rb
+test/test_model_country.rb
+test/test_model_region.rb
+test/test_models.rb

data/README.md

@@ -0,0 +1,85 @@
+# worlddb-models
+
+worlddb-models gem - world.db schema & models for easy (re)use
+
+* home  :: [github.com/worlddb/world.db.models](https://github.com/worlddb/world.db.models)
+* bugs  :: [github.com/worlddb/world.db.models/issues](https://github.com/worlddb/world.db.models/issues)
+* gem   :: [rubygems.org/gems/worlddb-models](https://rubygems.org/gems/worlddb-models)
+* rdoc  :: [rubydoc.info/gems/worlddb-models](http://rubydoc.info/gems/worlddb-models)
+* forum :: [groups.google.com/group/openmundi](https://groups.google.com/group/openmundi)
+
+
+## Usage Models
+
+`Country` Model - Example:
+
+    at = Country.find_by! key: 'at'
+    at.name
+    # => 'Austria'
+    at.pop
+    # => 8_414_638
+    at.area
+    # => 83_871
+    
+    at.regions.count
+    # => 9
+    at.regions
+    # => [ 'Wien', 'Niederösterreich', 'Oberösterreich', ... ]
+
+    at.cities.by_pop
+    # => [ 'Wien', 'Graz', 'Linz', 'Salzburg', 'Innsbruck' ... ]
+
+
+`City` Model - Example:
+
+    c = City.find_by! key: 'wien'
+    c.name
+    # => 'Wien'
+    c.country.name
+    # => 'Austria'
+    c.country.continent.name
+    # => 'Europe'
+
+    la = City.find_by! key: 'losangeles'
+    la.name
+    # => 'Los Angeles'
+    la.region.name
+    # => 'California'
+    la.region.key
+    # => 'ca'
+    la.country.name
+    # => 'United States'
+    la.country.key
+    # => 'us'
+    la.country.continent.name
+    # => 'North America'
+
+
+`Tag` Model - Example:
+
+    euro = Tag.find_by! key: 'euro'
+    euro.countries.count
+    # => 17
+    euro.countries
+    # => ['Austria, 'Belgium', 'Cyprus', ... ]
+    
+    flanders = Tag.find_by! key: 'flanders'
+    flanders.regions.count
+    # => 5
+    flanders.regions
+    # => ['Antwerpen', 'Brabant Wallon', 'Limburg', 'Oost-Vlaanderen', 'West-Vlaanderen']
+    flanders.regions.first.country.name
+    # => 'Belgium'
+
+and so on.
+
+
+## License
+
+The `worlddb-models` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+
+## Questions? Comments?
+
+Send them along to the [Open Mundi (world.db) Database Forum/Mailing List](http://groups.google.com/group/openmundi).
+Thanks!

data/Rakefile

@@ -0,0 +1,44 @@
+require 'hoe'
+require './lib/worlddb/version.rb'
+
+
+Hoe.spec 'worlddb-models' do
+  
+  self.version = WorldDb::VERSION
+
+  self.summary = "worlddb -  world.db schema & models for easy (re)use"
+  self.description = summary
+
+  self.urls    = ['https://github.com/worlddb/world.db.models']
+
+  self.author  = 'Gerald Bauer'
+  self.email   = 'openmundi@googlegroups.com'
+
+  self.extra_deps = [
+    ['props'],         # settings / prop(ertie)s / env / INI
+    ['logutils'],     # logging
+    ['textutils', '>= 0.9.9'], # e.g.  >= 0.6 && <= 1.0  ## will include logutils, props
+
+    ['tagutils'],     # tags n categories for activerecord
+    ['activerecord-utils'],
+    ['props-activerecord'],
+    ['logutils-activerecord'],
+
+     ## 3rd party
+    ['rubyzip'],       ## todo: pull in via textutils ??
+    ['activerecord']  # NB: will include activesupport,etc.
+  ]
+
+  # switch extension to .markdown for gihub formatting
+  #  -- NB: auto-changed when included in manifest
+  self.readme_file  = 'README.md'
+  self.history_file = 'HISTORY.md'
+
+
+  self.licenses = ['Public Domain']
+
+  self.spec_extras = {
+    required_ruby_version: '>= 1.9.2'
+  }
+
+end

data/lib/worlddb/deleter.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+module WorldDb
+
+  class Deleter
+    
+    ## make models available in worlddb module by default with namespace
+    #  e.g. lets you use City instead of Models::City 
+    include WorldDb::Models
+
+    def run
+      # for now delete all tables
+      
+      ## Tagging.delete_all   # - use TagDb.delete!
+      ## Tag.delete_all
+
+      CountryCode.delete_all
+      Name.delete_all
+      Place.delete_all
+      City.delete_all
+      Region.delete_all
+      Country.delete_all
+      Continent.delete_all
+      Usage.delete_all
+      Lang.delete_all
+
+      #  Prop.delete_all    # - use ConfDb.delete!
+    end
+
+  end # class Deleter
+
+end # module WorldDb

data/lib/worlddb/matcher.rb

@@ -0,0 +1,143 @@
+# encoding: UTF-8
+
+module WorldDb
+
+module Matcher
+
+  def match_xxx_for_country( name, xxx )  # xxx e.g. cities|regions|beers|breweries
+    #      auto-add required country code (from folder structure)
+    #  note: always let match_xxx_for_country_n_region go first
+
+    # note: allow  /cities and /1--hokkaido--cities
+    xxx_pattern = "(?:#{xxx}|[0-9]+--[^\\/]+?--#{xxx})"    # note: double escape \\ required for backslash
+
+    ##
+    ## todo: add $-anchor at the end of pattern - why? why not?? (will include .txt or .yaml??)
+
+    if name =~ /(?:^|\/)([a-z]{2,3})-[^\/]+\/#{xxx_pattern}/         ||    # (1)
+       name =~ /(?:^|\/)[0-9]+--([a-z]{2,3})-[^\/]+\/#{xxx_pattern}/ ||    # (2)
+       name =~ /(?:^|\/)([a-z]{2,3})\/#{xxx_pattern}/                ||    # (3)
+       name =~ /(?:^|\/)([a-z]{2,3})-[^\/]+\/[0-9]+--[^\/]+\/#{xxx_pattern}/ ||   # (4)
+       name =~ /(?:^|\/)([a-z]{2,3})-[^\/]+--#{xxx}/    # (5)
+
+      country_key = $1.dup
+      yield( country_key )
+      true # bingo - match found
+      
+      ######
+      # (1)  new style: e.g. /at-austria/beers or ^at-austria!/cities
+      #
+      # (2)  new-new style e.g. /1--at-austria--central/cities
+      #
+      # (3)  classic style: e.g. /at/beers (europe/at/cities)
+      #
+      # (4) new style w/ region w/o abbrev/code e.g. /ja-japon/1--hokkaido/cities
+      #
+      # (5)  compact style (country part of filename):
+      #   e.g. /at-austria--cities or /europe/at-austria--cities
+    else
+      false # no match found
+    end
+  end
+
+
+  def match_xxx_for_country_n_region( name, xxx ) # xxx e.g. wine|wineries
+
+    # auto-add required country n region code (from folder structure)
+
+    ## -- allow opt_folders after long regions (e.g. additional subregion/zone)
+    ## -- allow anything (prefixes) before -- for xxx
+    #       e.g.  at-austria!/1--n-niederoesterreich--eastern/wagram--wines
+    #             at-austria!/1--n-niederoesterreich--eastern/wagram--wagram--wines
+
+    # note: allow  /cities and /1--hokkaido--cities and /hokkaido--cities too
+    xxx_pattern = "(?:#{xxx}|[^\\/]+--#{xxx})"    # note: double escape \\ required for backslash
+    
+    ## allow optional folders -- TODO: add restriction ?? e.g. must be 4+ alphas ???
+    opt_folders_pattern = "(?:\/[^\/]+)*"
+    ## note: for now only (style #2) n (style #3)  that is long region allow opt folders
+
+    if name =~ /(?:^|\/)([a-z]{2,3})-[^\/]+\/([a-z]{1,3})-[^\/]+\/#{xxx_pattern}/  ||                # (1)
+       name =~ /(?:^|\/)[0-9]+--([a-z]{2,3})-[^\/]+\/[0-9]+--([a-z]{1,3})-[^\/]+#{opt_folders_pattern}\/#{xxx_pattern}/ || # (2)
+       name =~ /(?:^|\/)([a-z]{2,3})-[^\/]+\/[0-9]+--([a-z]{1,3})-[^\/]+#{opt_folders_pattern}\/#{xxx_pattern}/         || # (3)
+       name =~ /(?:^|\/)[0-9]+--([a-z]{2,3})-[^\/]+\/([a-z]{1,3})-[^\/]+\/#{xxx_pattern}/            # (4)
+
+      #######
+      # nb: country must start name (^) or coming after / e.g. europe/at-austria/...
+      # (1)
+      # new style: e.g.  /at-austria/w-wien/cities or
+      #                  ^at-austria!/w-wien/cities
+      # (2)
+      # new new style e.g.  /1--at-austria--central/1--w-wien--eastern/cities
+      #
+      # (3)
+      #  new new mixed style e.g.  /at-austria/1--w-wien--eastern/cities
+      #      "classic" country plus new new region
+      #
+      # (4)
+      #  new new mixed style e.g.  /1--at-austria--central/w-wien/cities
+      #      new new country plus "classic" region
+
+      country_key = $1.dup
+      region_key  = $2.dup
+      yield( country_key, region_key )
+      true # bingo - match found
+    else
+      false # no match found
+    end
+  end
+
+
+  def match_cities_for_country( name, &blk )
+    ## todo: check if there's a better (more ruby way) to pass along code block ??
+    ## e.g. try
+    ##   match_xxx_for_country( name, 'cities') { |country_key| yield(country_key) }
+
+    match_xxx_for_country( name, 'cities', &blk )
+  end
+
+  def match_regions_for_country( name, &blk )
+    ## also try synonyms e.g. old regions (if not match for states)
+    found = match_xxx_for_country( name, 'states', &blk )
+    found = match_xxx_for_country( name, 'regions', &blk ) unless found
+    found
+  end
+
+  def match_regions_abbr_for_country( name, &blk )  # NB: . gets escaped for regex, that is, \.
+    ## also try synonyms e.g. old regions (if not match for states)
+    found = match_xxx_for_country( name, 'states\.abbr', &blk )
+    found = match_xxx_for_country( name, 'regions\.abbr', &blk ) unless found
+    found
+  end
+
+  def match_regions_iso_for_country( name, &blk )  # NB: . gets escaped for regex, that is, \.
+    ## also try synonyms e.g. old regions (if not match for states)
+    found = match_xxx_for_country( name, 'states\.iso', &blk )
+    found = match_xxx_for_country( name, 'regions\.iso', &blk ) unless found
+    found 
+  end
+
+  def match_regions_nuts_for_country( name, &blk )  # NB: . gets escaped for regex, that is, \.
+    ## also try synonyms e.g. old regions (if not match for states)
+    found = match_xxx_for_country( name, 'states\.nuts', &blk )
+    found = match_xxx_for_country( name, 'regions\.nuts', &blk ) unless found
+    found
+  end
+
+
+  def match_countries_for_continent( name )
+    if name =~ /^([a-z][a-z\-_]+[a-z])\/countries/     # e.g. africa/countries or america/countries
+      ### NB: continent changed to regions (e.g. middle-east, caribbean, north-america, etc.)
+      ## auto-add continent (from folder structure) as tag
+      ## fix: allow dash/hyphen/minus in tag
+      continent = $1.dup
+      yield( continent )
+      true
+    else
+      false # no match found
+    end
+  end
+
+end # module Matcher
+
+end # module WorldDb

data/lib/worlddb/models/city.rb

@@ -0,0 +1,240 @@
+# encoding: utf-8
+
+module WorldDb
+  module Model
+
+###
+##  Todo:
+##  use four classes instead of one ?
+#    e.g. Use class class Metro n class City n class District n class CityBase ?? - why? why not?
+#
+#  find a better name for CityBase ??
+#     Locality ??
+#      or CityCore or CityStd or CityAll or CityGeneric
+#      or CityLike or CityTable or CityTbl or ???
+
+class City < ActiveRecord::Base
+  
+  extend TextUtils::TagHelper  # will add self.find_tags, self.find_tags_in_attribs!, etc.
+
+  # NB: use extend - is_<type>? become class methods e.g. self.is_<type>? for use in
+  #   self.create_or_update_from_values
+  extend TextUtils::ValueHelper  # e.g. self.is_year?, self.is_region?, self.is_address?, self.is_taglist? etc.
+
+
+  self.table_name = 'cities'
+
+  belongs_to :place,   class_name: 'Place',   foreign_key: 'place_id'
+  belongs_to :country, class_name: 'Country', foreign_key: 'country_id'
+  belongs_to :region,  class_name: 'Region',  foreign_key: 'region_id'
+
+  ## self referencing hierachy within cities e.g. m|metro > c|city > d|district
+
+  ## fix: use condition check for m|d|c flag?? why? why not? (NB: flags are NOT exclusive e.g. possible metro|city)
+  
+  ## (1) metro - level up
+  has_many   :cities,    class_name: 'City',  foreign_key: 'city_id'
+
+  ## (2) city
+  belongs_to :metro,     class_name: 'City',  foreign_key: 'city_id'   ## for now alias for parent - use parent?
+  has_many   :districts, class_name: 'City',  foreign_key: 'city_id'   ## for now alias for cities - use cities?
+
+  ## (3) district - level down
+  belongs_to :city,      class_name: 'City',  foreign_key: 'city_id'  ## for now alias for parent - use parent?
+
+  has_many_tags
+
+  ###
+  #  NB: use is_  for flags to avoid conflict w/ assocs (e.g. metro?, city? etc.)
+  
+  def is_metro?()    m? == true;  end
+  def is_city?()     c? == true;  end
+  def is_district?() d? == true;  end
+
+  before_create :on_create
+  before_update :on_update
+
+  def on_create
+    place_rec = Place.create!( name: name, kind: place_kind )
+    self.place_id = place_rec.id 
+  end
+
+  def on_update
+    ## fix/todo: check - if name or kind changed - only update if changed ?? why? why not??
+    place.update_attributes!( name: name, kind: place_kind )
+  end
+
+  def place_kind   # use place_kind_of_code ??
+    ### fix/todo: make sure city records won't overlap (e.g. using metro n city flag at the same time; use separate records)
+#//////////////////////////////////
+#// fix: add nested record syntax e.g. city w/ metro population
+#//  use (metro: 4444)  e.g. must start with (<nested_type>: props) !!! or similar
+#//
+    if is_metro?
+      'MTRO'
+    elsif is_district?
+      'DIST'
+    else
+      'CITY'
+    end
+  end
+
+
+  validates :key,  format: { with: /#{CITY_KEY_PATTERN}/, message: CITY_KEY_PATTERN_MESSAGE }
+  validates :code, format: { with: /#{CITY_CODE_PATTERN}/, message: CITY_CODE_PATTERN_MESSAGE }, allow_nil: true
+
+
+  scope :by_key,   ->{ order( 'key asc' )  }  # order by key (a-z)
+  scope :by_name,  ->{ order( 'name asc' ) } # order by title (a-z)
+  scope :by_pop,   ->{ order( 'pop desc' ) }  # order by pop(ulation)
+  scope :by_popm,  ->{ order( 'popm desc' ) } # order by pop(ulation) metropolitan area
+  scope :by_area,  ->{ order( 'area desc' ) }  # order by area (in square km)
+
+
+  def all_names( opts={} )
+    ### fix:
+    ## allow to passing in sep or separator e.g. | or other
+
+    return name if alt_names.blank?
+    
+    buf = ''
+    buf << name
+    buf << ' | '
+    buf << alt_names.split('|').join(' | ')
+    buf
+  end
+
+
+  def self.create_or_update_from_values( values, more_attribs={} )
+    ## key & title & country required
+
+    attribs, more_values = find_key_n_title( values )
+    attribs = attribs.merge( more_attribs )
+
+    ## check for optional values
+    City.create_or_update_from_attribs( attribs, more_values )
+  end
+
+
+  def self.create_or_update_from_titles( titles, more_attribs = {} )
+    # ary of titles e.g. ['Wien', 'Graz'] etc.
+
+    titles.each do |title|
+      values = [title]
+      City.create_or_update_from_values( values, more_attribs ) 
+    end # each city
+  end  # method create_or_update_from_titles
+
+
+
+  def self.create_or_update_from_attribs( new_attributes, values, opts={} )
+    #   attribs -> key/value pairs e.g. hash
+    #   values  -> ary of string values/strings (key not yet known; might be starting of value e.g. city:wien)
+
+    ## opts e.g. :skip_tags true|false
+
+    ## fix: add/configure logger for ActiveRecord!!!
+    logger = LogKernel::Logger.root
+
+    value_numbers     = []
+    value_tag_keys    = []
+      
+    ### check for "default" tags - that is, if present new_attributes[:tags] remove from hash
+    value_tag_keys += find_tags_in_attribs!( new_attributes )
+
+    new_attributes[ :c ] = true   # assume city type by default (use metro,district to change in fixture)
+
+    ## check for optional values
+
+    values.each_with_index do |value,index|
+      if match_region_for_country( value, new_attributes[:country_id] ) do |region|
+           new_attributes[ :region_id ] = region.id
+         end
+      elsif match_country( value ) do |country|
+              new_attributes[ :country_id ] = country.id
+            end
+      elsif match_metro( value ) do |city|
+              new_attributes[ :city_id ] = city.id
+            end
+      elsif match_metro_pop( value ) do |num|  # m:
+              new_attributes[ :popm ] = num
+              new_attributes[ :m ] = true   #  auto-mark city as m|metro too
+            end
+      elsif match_metro_flag( value ) do |_|  # metro(politan area)
+              new_attributes[ :c ] = false   # turn off default c|city flag; make it m|metro only
+              new_attributes[ :m ] = true
+            end
+      elsif match_city( value ) do |city|  # parent city for district
+              new_attributes[ :city_id ] = city.id
+              new_attributes[ :c ] = false # turn off default c|city flag; make it d|district only
+              new_attributes[ :d ] = true
+            end
+      elsif match_km_squared( value ) do |num|   # allow numbers like 453 km²
+              value_numbers << num
+            end
+      elsif match_number( value ) do |num|    # numeric (nb: can use any _ or spaces inside digits e.g. 1_000_000 or 1 000 000)
+              value_numbers << num
+            end
+      elsif value =~ /#{CITY_CODE_PATTERN}/  ## assume three-letter code
+        new_attributes[ :code ] = value
+      elsif (values.size==(index+1)) && is_taglist?( value )   # tags must be last entry
+        logger.debug "   found tags: >>#{value}<<"
+        value_tag_keys += find_tags( value )
+      else
+        # issue warning: unknown type for value
+        logger.warn "unknown type for value >#{value}<"
+      end
+    end # each value
+
+    if value_numbers.size > 0
+      new_attributes[ :pop  ] = value_numbers[0]   # assume first number is pop for cities
+      new_attributes[ :area ] = value_numbers[1]  
+    end
+
+    rec = City.find_by_key( new_attributes[ :key ] )
+
+    if rec.present?
+      logger.debug "update City #{rec.id}-#{rec.key}:"
+    else
+      logger.debug "create City:"
+      rec = City.new
+    end
+
+    logger.debug new_attributes.to_json
+
+    rec.update_attributes!( new_attributes )
+
+      ##################
+      ## add taggings
+
+      ## todo/fix: reuse - move add taggings into method etc.
+
+      if value_tag_keys.size > 0
+
+        if opts[:skip_tags].present?
+          logger.debug "   skipping add taggings (flag skip_tag)"
+        else
+          value_tag_keys.uniq!  # remove duplicates
+          logger.debug "   adding #{value_tag_keys.size} taggings: >>#{value_tag_keys.join('|')}<<..."
+
+          ### fix/todo: check tag_ids and only update diff (add/remove ids)
+
+          value_tag_keys.each do |key|
+            tag = Tag.find_by_key( key )
+            if tag.nil?  # create tag if it doesn't exit
+              logger.debug "   creating tag >#{key}<"
+              tag = Tag.create!( key: key )
+            end
+            rec.tags << tag
+          end
+        end
+      end
+
+    rec
+  end # method create_or_update_from_values
+
+
+end # class Cities
+
+  end # module Model
+end # module WorldDb

data/lib/worlddb/models/city_comp.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+module WorldDb
+  module Model
+
+#############################################################
+# collect depreciated or methods for future removal here
+#  - keep for now for commpatibility (for old code)
+
+class City
+
+  def title()       name;              end
+  def title=(value) self.name = value; end
+
+  scope :by_title, ->{ order( 'name asc' ) } # order by title (a-z)
+
+
+  def synonyms()       alt_names;      end
+  def synonyms=(value) self.alt_names = value; end
+
+  def title_w_synonyms( opts={} )  all_names( opts );  end    # depreciated: use all_names instead
+
+end # class Cities
+
+  end # module Model
+end # module WorldDb
+

data/lib/worlddb/models/continent.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+module WorldDb
+  module Model
+
+class Continent < ActiveRecord::Base
+
+  belongs_to :place,     class_name: 'Place',   foreign_key: 'place_id'
+  has_many   :countries
+
+  # NB: allow dots in keys e.g. concacaf.naf etc.    
+
+  before_create :on_create
+  before_update :on_update
+
+  def on_create
+    place_rec = Place.create!( name: name, kind: place_kind )
+    self.place_id = place_rec.id 
+
+    self.slug = TextUtils.slugify( name )  if slug.blank?
+  end
+
+  def on_update
+    ## fix/todo: check - if name or kind changed - only update if changed ?? why? why not??
+    place.update_attributes!( name: name, kind: place_kind )
+    
+    ## check if name changed -- possible?
+    ## update slug too??
+  end
+
+  def place_kind   # use place_kind_of_code ??
+    'CONT'
+  end
+
+
+end  # class Continent
+
+
+  end # module Model
+end # module WorldDb
+