worlddb-models 2.1.0

data/lib/worlddb/readers/lang.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class LangReader
+
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  ## todo: add opts={} etc.
+  def self.from_zip( zip_file, entry_path )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text )
+  end
+
+  def self.from_file( path, opts={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, opts )
+  end
+
+  def self.from_string( text, opts={} )
+    LangReader.new( text, opts )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, opts={} )
+    @text = text
+
+    ## option: do NOT generate/add any tags for countries/regions/cities
+    @skip_tags =  opts[:skip_tags].present? ? true : false
+    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
+    @strict    =  opts[:strict].present? ? true : false
+  end
+
+  def read()
+    reader = HashReader.from_string( @text )
+
+    reader.each do |key, value|
+
+      ### fix:
+      ##  move to Lang.read() for (re)use
+
+        logger.debug "adding lang >>#{key}<< >>#{value}<<..."
+      
+        lang_key   = key.strip
+        lang_title = value.strip
+      
+        lang_attribs = {}
+        
+        ## check if it exists
+        lang = Lang.find_by_key( lang_key )
+        if lang.present?
+          logger.debug "update lang #{lang.id}-#{lang.key}:"
+        else
+          logger.debug "create lang:"
+          lang = Lang.new
+          lang_attribs[ :key ] = lang_key
+        end
+        
+        lang_attribs[ :title ] = lang_title
+        
+        logger.debug lang_attribs.to_json
+     
+        lang.update_attributes!( lang_attribs )
+    end # each key,value
+
+  end # method load_langs
+
+
+end # class LangReader
+end # module WorldDb

data/lib/worlddb/readers/region.rb

@@ -0,0 +1,79 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class RegionReader
+
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  def self.from_zip( zip_file, entry_path, more_attribs={} )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_file( path, more_attribs={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_string( text, more_attribs={} )
+    RegionReader.new( text, more_attribs )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, more_attribs={} )
+    ## todo/fix: how to add opts={} ???
+
+    @text         = text
+    @more_attribs = more_attribs
+  end
+
+
+  def read()
+    reader = ValuesReader.from_string( @text, @more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      Region.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class RegionReader
+end # module WorldDb

data/lib/worlddb/readers/usage.rb

@@ -0,0 +1,98 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class UsageReader
+
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+  ## todo: add opts
+  def self.from_zip( zip_file, entry_path )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text )
+  end
+
+  def self.from_file( path, opts={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, opts )
+  end
+
+  def self.from_string( text, opts={} )
+    UsageReader.new( text, opts )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, opts={} )
+    @text = text
+
+    ## option: do NOT generate/add any tags for countries/regions/cities
+    @skip_tags =  opts[:skip_tags].present? ? true : false
+    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
+    @strict    =  opts[:strict].present? ? true : false
+  end
+
+  def read()
+    reader = HashReader.from_string( @text )
+
+    reader.each do |key, value|
+      
+      ### fix:
+      ##  move to Usage.read() for (re)use
+
+      logger.debug "   adding langs >>#{value}<<to country >>#{key}<<"
+      
+      country = Country.find_by_key!( key )
+      
+      lang_keys = value.split(',')
+      lang_keys.each do |lang_key|
+
+        ### remove (optional comment) from key (e.g. carribean (islands))
+        lang_key = lang_key.gsub( /\(.+\)/, '' )
+        ## remove leading n trailing space
+        lang_key = lang_key.strip
+
+        lang = Lang.find_by_key!( lang_key )
+        Usage.create!( country_id: country.id, lang_id: lang.id, official: true, minor: false )
+      end
+    end
+  end
+
+
+end # class UsageReader
+end # module WorldDb

data/lib/worlddb/schema.rb

@@ -0,0 +1,202 @@
+
+module WorldDb
+
+class CreateDb 
+
+def up
+
+ActiveRecord::Schema.define do
+
+create_table :places do |t|
+  t.string  :name,   null: false
+  t.string  :kind,   null: false  # --  kind/feature (note: type reserved for activerecord sti)
+  #####
+  # continent:
+  #  CONT - continent (use CNTI or CENT why??)
+  # country:
+  #  SUPR - supra (e.g. European Union)
+  #  CNTY - country
+  #  TERR - terr
+  # region:
+  #  ADM1 - e.g. region/state/province
+  #  ADM2 - e.g. county/district/
+  #  ADM3 - e.g. 
+  # city:
+  #  MTRO - metro
+  #  CITY - city/town/
+  #  DIST - district/
+  #
+  #  add new table for zones (e.g. informal regions e.g. tourism, wine regions, etc.) ??
+  #   why? why not??
+
+
+  t.float   :lat   # optional for now (latitude)
+  t.float   :lng   # optional for now (longitude)
+
+  ## todo: add parent for hierachy ?? or keep it in country/region/city etc. table ??
+
+  ## timestamp at last
+  t.timestamps
+end
+
+
+#############
+# todo: use many-to-many assoc w/ join table  for name and place ???
+#  why? why not?
+#    - wien -> city n region n metro ?  collect more samples of names used more than once
+
+### alternative names for places
+create_table :names do |t|
+  t.string     :name,   null: false
+  t.references :place,  null: false
+  ### todo/fix: add lang_id to reference lang from lang table!!! (for now make it a duplicate; e.g. keep stirng lang for now)
+  t.string     :lang,   null: false, default: 'en'   # default to english for now (or use unknown/undeterminded/unspecified???)
+
+  ## todo: add kind/type   ? e.g. postal for postal code ??
+  ## or wikipedia for wikipedia?  or use separate table ?? (linkable/links) etc.??
+
+  ## timestamp at last
+  t.timestamps
+end
+
+
+
+create_table :continents do |t|
+  t.string     :name,     null: false
+  t.string     :slug,     null: false   # auto-generate default
+  t.string     :key,      null: false
+  t.references :place,    null: false
+  t.string     :alt_names  # comma separated list of alternate names (synonyms)
+
+  ## timestamp at last
+  t.timestamps
+end
+
+add_index :continents, :key, unique: true
+
+
+create_table :country_codes do |t|
+  t.string     :name, null: false
+  t.string     :kind, null: false   # e.g. ALPHA2,NUM,FIFA,IOC,FIPS,NET,etc.
+  t.references :country, null: false
+end
+
+add_index :country_codes, [:name, :kind], unique: true
+
+
+create_table :countries do |t|
+  t.string     :name,   null: false
+  t.string     :slug,   null: false   # auto-generate default
+  t.string     :key,    null: false
+  t.references :place,  null: false
+  t.string     :code,   null: false  # short three letter code (FIFA country code e.g. ITA)
+  t.string     :alt_names  # comma separated list of alternate names (synonyms)
+  t.string     :hist_names # comma separated list of historic names (no longer in use)
+  t.integer    :pop,    null: false    # population count
+  t.integer    :area,   null: false    #  area in square km (sq. km)
+  t.references :continent
+  t.references :country    # for supra(nationals) n depend(encies)
+
+  ## flags (use single int named flags - why? why not?  
+  t.boolean :s,  null: false, default: false   # supra(national) flag e.g. eu
+  t.boolean :c,  null: false, default: false   # country flag (is this needed?)
+  t.boolean :d,  null: false, default: false   # dependency flag
+
+  t.boolean :m,  null: false, default: false   # misc(ellaneous) flag
+  # misc use for territories w/ shared or disputed claims
+  #   e.g. Antartica/Western Sahara/Paracel Islands/Spratly Islands/etc.
+
+  # extras - country codes
+  t.string  :motor      # optional  motor (vehicle) licene plate code (bumper sticker)
+  t.string  :alpha2       # optional iso two letter country code
+  t.string  :alpha3       # optional iso three letter country code
+  t.string  :num        #  optional iso numeric country code - NOTE: same numeric code as string!!! e.g. 043 etc.
+  t.string  :fifa       # optional fifa country code
+  t.string  :ioc
+  t.string  :fips
+  t.string  :net        # optional internet top level domain (tld)
+
+  t.string  :wikipedia  # optional wikipedia page name -- en.wikipedia.org/wiki/<name>
+
+  ## timestamp at last
+  t.timestamps
+end
+
+
+add_index :countries, :key,  unique: true
+add_index :countries, :code, unique: true
+
+
+######
+# NB: rename to adms/admins ??
+#
+# used for state/provice/land/regioni/etc.
+create_table :regions do |t|
+  t.string     :name,   null: false
+  t.string     :key,    null: false
+  t.references :place,  null: false
+  t.string :code     # short two or three letter code e.g. NY, OAX, etc.
+  t.string :abbr     # optional conventional abbrevation (e.g. Stmk., Gto., etc.)
+  t.string :iso      # iso code
+  t.string :nuts     # nuts code (europe/eu only)
+  t.string     :alt_names  # comma separated list of alternate names (synonyms)
+  t.references :country,  null: false
+  t.integer :pop     # optional population count
+  t.integer :area    # optional area in square km (sq. km)
+  t.timestamps
+end
+
+add_index :regions, [:key, :country_id], unique: true
+
+
+create_table :cities do |t|
+  t.string     :name,   null: false
+  t.string     :key,    null: false
+  t.references :place,  null: false
+  t.string     :code     # short three letter code (ITAT/airport code e.g. NYC or VIE)
+  t.string     :alt_names  # comma separated list of alternate names (synonyms)
+  t.references :country,  null: false
+  t.references :region    # optional for now
+  t.references :city  # optional parent (e.g. metro for city, or city for district)
+  t.integer :pop     # optional population count (city proper)
+  t.integer :popm    # optional population count (metropolitan/aglomeration)
+  t.integer :area    # optional area in square km (sq. km)
+
+  ## t.float   :lat   # optional for now  --  FIX: remove?? moved to places
+  ## t.float   :lng   # optional for now  --  FIX: remove?? moved to places
+
+  ## flags (use single int named flags - why? why not?    
+  t.boolean :m,  null: false, default: false   # metro flag
+  t.boolean :c,  null: false, default: false   # city flag (is this needed?)
+  t.boolean :d,  null: false, default: false   # district flag
+
+  ### t.boolean :capital, null: false, default: false  # is national captial?
+
+  t.timestamps
+end
+
+
+create_table :langs do |t|  # langs == languages (e.g. en/English, de/Deutsch, etc.)
+  t.string :key,   null: false
+  t.string :name,  null: false
+  t.timestamps
+end
+
+
+create_table :usages do |t|  # join table for countries_langs
+  t.references :country,  null: false
+  t.references :lang,     null: false
+  t.boolean    :official, null: false, default: true  # is_official language in country
+  t.boolean    :minor,    null: false, default: false # spoken by minority 
+  t.float      :percent     # usage in percent e.g. 90.0, 0.55, etc.
+  t.timestamps
+end
+
+end # block Schema.define
+
+end # method up
+
+
+end # class CreateDb
+
+end # module WorldDb

data/lib/worlddb/stats.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+module WorldDb
+
+  class Stats
+    include Models
+
+    def tables
+      puts "Stats:"
+      puts " #{'%5d' % Continent.count} continents"
+      puts " #{'%5d' % Country.count} countries (#{Country.where(s: true).count} supras, #{Country.where(d:true).count} deps), #{Country.where(m:true).count} miscs)"
+      puts " #{'%5d' % Region.count} regions"
+      puts " #{'%5d' % City.where(m: true).where(c: false).count} metros"
+      puts " #{'%5d' % City.where(c: true).count} cities (#{City.where(c: true).where(m: true).count} metros)"
+      puts " #{'%5d' % City.where(d: true).count} districts"
+      puts " #{'%5d' % Place.count} places"
+      puts " #{'%5d' % Name.count} names"
+
+      puts " #{'%5d' % Lang.count} langs"
+      puts " #{'%5d' % Usage.count} usages"
+
+      puts " #{'%5d' % CountryCode.count} (country) codes"
+
+#      puts " #{'%5d' % Tag.count} tags"
+#      puts " #{'%5d' % Tagging.count} taggings"
+    end
+
+  end # class Stats
+
+end # module WorldDb
+

data/lib/worlddb/version.rb

@@ -0,0 +1,23 @@
+
+module WorldDb
+
+  # sync version w/ sport.db n friends - why? why not?
+  MAJOR = 2 ## todo: namespace inside version or something - why? why not??
+  MINOR = 1
+  PATCH = 0
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "worlddb-models/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}"
+  end
+
+end
+

data/test/helper.rb

@@ -0,0 +1,26 @@
+
+## $:.unshift(File.dirname(__FILE__))
+
+## minitest setup
+require 'minitest/autorun'
+
+
+# our own code
+
+require 'worlddb/models'
+
+
+Name      = WorldDb::Model::Name
+Place     = WorldDb::Model::Place
+Continent = WorldDb::Model::Continent
+Country   = WorldDb::Model::Country
+Region    = WorldDb::Model::Region
+City      = WorldDb::Model::City
+
+Lang      = WorldDb::Model::Lang
+Usage     = WorldDb::Model::Usage
+
+
+
+WorldDb.setup_in_memory_db
+

data/test/test_fixture_matchers.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+
+require 'helper'
+
+
+class TestFixtureMatchers < MiniTest::Test
+
+  include WorldDb::Matcher
+
+
+  def  test_cities_for_country
+
+    cities_at = [
+      'europe/at/cities',
+      'europe/at-austria/cities',
+      'at-austria/cities',
+      'at-austria!/cities',
+      'at-austria!/1--niederoesterreich--eastern--cities',
+      'at-austria!/1--niederoesterreich--eastern/cities',   # region w/o code/abbrev
+      'at-austria!/1--niederoesterreich--eastern/1--waldviertel--cities'
+    ]
+
+    cities_at.each do |name|
+      found = match_cities_for_country( name ) do |country_key|
+        assert_equal country_key, 'at'
+      end
+      assert found
+    end
+  end # method test_cities_for_country
+
+  def test_country
+
+    beers_at = [
+      'europe/at/beers',
+      'europe/at-austria/beers',
+      'europe/at-austria--beers',
+      'at-austria/beers',
+      'at-austria!/beers',
+      '1--at-austria--central/beers',
+      '1--at-austria--central/1--niederoesterreich--eastern--beers',
+      'europe/1--at-austria--central/beers',
+      'europe/1--at-austria--central/1--niederosterreich--eastern--beers',
+      'at-austria!/1--niederoesterreich--eastern--beers',
+      'at-austria!/1--niederoesterreich--eastern/beers',    # region w/o code/abbrev
+      'at-austria!/1--niederoesterreich--eastern/1--waldviertel--beers'
+    ]
+
+    beers_at.each do |name|
+      found = match_xxx_for_country( name, 'beers' ) do |country_key|
+        assert_equal country_key, 'at'
+      end
+      assert found, "no match found for '#{name}'"
+    end
+
+    breweries_at = [
+      'europe/at/breweries',
+      'europe/at-austria/breweries',
+      'at-austria/breweries',
+      'at-austria!/breweries'
+    ]
+
+    breweries_at.each do |name|
+      found = match_xxx_for_country( name, 'breweries' ) do |country_key|
+        assert_equal country_key, 'at'
+      end
+      assert found
+    end
+  end # method test_country
+
+
+  def test_country_n_region
+
+    beers_at = [
+      'europe/at-austria/w-wien/beers',
+      'at-austria/w-wien/beers',
+      'at-austria/w-wien/2--leopoldstadt--beers',
+      'at-austria!/w-wien/beers',
+      '1--at-austria--central/1--w-wien--eastern/beers',
+      '1--at-austria--central/1--w-wien--eastern/2--leopoldstadt--beers',
+      'europe/1--at-austria--central/1--w-wien--eastern/beers',
+      'at-austria/1--w-wien--eastern/beers',
+      'europe/at-austria!/1--w-wien--eastern/beers',
+      '1--at-austria--central/w-wien/beers',
+      'europe/1--at-austria--central/w-wien/beers',
+      'europe/1--at-austria--central/w-wien/2--leopoldstadt--beers'
+    ]
+
+    beers_at.each do |name|
+      found = match_xxx_for_country_n_region( name, 'beers' ) do |country_key,region_key|
+        assert_equal country_key, 'at'
+        assert_equal region_key,  'w'
+      end
+      assert found
+    end
+
+    breweries_at = [
+      'europe/at-austria/w-wien/breweries',
+      'at-austria/w-wien/breweries',
+      'at-austria!/w-wien/breweries'
+    ]
+
+    breweries_at.each do |name|
+      found = match_xxx_for_country_n_region( name, 'breweries' ) do |country_key,region_key|
+        assert_equal country_key, 'at'
+        assert_equal region_key,  'w'
+      end
+      assert found
+    end
+  end # method test_country_n_region
+
+
+end # class TestFixtureMatchers