worlddb 1.8.2 → 2.0.0

data/lib/worlddb/readers/base.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+
+module WorldDb
+
+
+class BaseReader
+
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  attr_reader :include_path
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+
+  def initialize( include_path, opts = {} )
+    
+    @include_path = include_path
+    
+    ## option: do NOT generate/add any tags for countries/regions/cities
+    @skip_tags =  opts[:skip_tags].present? ? true : false
+    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
+    @strict    =  opts[:strict].present? ? true : false
+  end
+
+
+  def read( name, more_attribs={} )
+    puts "error: overwrite in concrete reader class!!!"  ### overwrite!!!!
+  end
+
+
+end # class BaseReader
+end # module WorldDb

data/lib/worlddb/readers/city.rb

@@ -0,0 +1,18 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class CityReader < BaseReader
+
+  def read( name, more_attribs={} )
+    reader = ValuesReaderV2.new( name, include_path, more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      City.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class CityReader
+end # module WorldDb
+

data/lib/worlddb/readers/country.rb

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class CountryReader < BaseReader
+
+  def read( name, more_attribs={} )
+    reader = ValuesReaderV2.new( name, include_path, more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      Country.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class CountryReader
+end # module WorldDb

data/lib/worlddb/readers/lang.rb

@@ -0,0 +1,43 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class LangReader < BaseReader
+
+  def read( name ) 
+    reader = HashReaderV2.new( name, include_path )
+
+    reader.each do |key, value|
+
+      ### fix:
+      ##  move to Lang.read() for (re)use
+
+        logger.debug "adding lang >>#{key}<< >>#{value}<<..."
+      
+        lang_key   = key.strip
+        lang_title = value.strip
+      
+        lang_attribs = {}
+        
+        ## check if it exists
+        lang = Lang.find_by_key( lang_key )
+        if lang.present?
+          logger.debug "update lang #{lang.id}-#{lang.key}:"
+        else
+          logger.debug "create lang:"
+          lang = Lang.new
+          lang_attribs[ :key ] = lang_key
+        end
+        
+        lang_attribs[ :title ] = lang_title
+        
+        logger.debug lang_attribs.to_json
+     
+        lang.update_attributes!( lang_attribs )
+    end # each key,value
+
+  end # method load_langs
+
+
+end # class LangReader
+end # module WorldDb

data/lib/worlddb/readers/region.rb

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class RegionReader < BaseReader
+
+  def read( name, more_attribs={} )
+    reader = ValuesReaderV2.new( name, include_path, more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      Region.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class RegionReader
+end # module WorldDb

data/lib/worlddb/readers/usage.rb

@@ -0,0 +1,35 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class UsageReader < BaseReader
+
+  def read( name )
+    reader = HashReaderV2.new( name, include_path )
+
+    reader.each do |key, value|
+      
+      ### fix:
+      ##  move to Usage.read() for (re)use
+
+      logger.debug "   adding langs >>#{value}<<to country >>#{key}<<"
+      
+      country = Country.find_by_key!( key )
+      
+      lang_keys = value.split(',')
+      lang_keys.each do |lang_key|
+
+        ### remove (optional comment) from key (e.g. carribean (islands))
+        lang_key = lang_key.gsub( /\(.+\)/, '' )
+        ## remove leading n trailing space
+        lang_key = lang_key.strip
+
+        lang = Lang.find_by_key!( lang_key )
+        Usage.create!( country_id: country.id, lang_id: lang.id, official: true, minor: false )
+      end
+    end
+  end
+
+
+end # class UsageReader
+end # module WorldDb

data/lib/worlddb/schema.rb

@@ -5,33 +5,90 @@ class CreateDb < ActiveRecord::Migration
 
 def up
 
+
+create_table :places do |t|
+  t.string  :name,   null: false
+  t.string  :kind,   null: false  # --  kind/feature (note: type reserved for activerecord sti)
+  #####
+  # continent:
+  #  CONT - continent (use CNTI or CENT why??)
+  # country:
+  #  SUPR - supra (e.g. European Union)
+  #  CNTY - country
+  #  TERR - terr
+  # region:
+  #  ADM1 - e.g. region/state/province
+  #  ADM2 - e.g. county/district/
+  #  ADM3 - e.g. 
+  # city:
+  #  MTRO - metro
+  #  CITY - city/town/
+  #  DIST - district/
+  #
+  #  add new table for zones (e.g. informal regions e.g. tourism, wine regions, etc.) ??
+  #   why? why not??
+
+
+  t.float   :lat   # optional for now (latitude)
+  t.float   :lng   # optional for now (longitude)
+
+  ## todo: add parent for hierachy ?? or keep it in country/region/city etc. table ??
+
+  ## timestamp at last
+  t.timestamps
+end
+
+
+#############
+# todo: use many-to-many assoc w/ join table  for name and place ???
+#  why? why not?
+#    - wien -> city n region n metro ?  collect more samples of names used more than once
+
+### alternative names for places
+create_table :names do |t|
+  t.string     :name,   null: false
+  t.references :place,  null: false
+  ### todo/fix: add lang_id to reference lang from lang table!!! (for now make it a duplicate; e.g. keep stirng lang for now)
+  t.string     :lang,   null: false, default: 'en'   # default to english for now (or use unknown/undeterminded/unspecified???)
+
+  ## todo: add kind/type   ? e.g. postal for postal code ??
+  ## or wikipedia for wikipedia?  or use separate table ?? (linkable/links) etc.??
+
+  ## timestamp at last
+  t.timestamps
+end
+
+
+
 create_table :continents do |t|
-  t.string  :title,  :null => false
-  t.string  :key,    :null => false
-  t.string  :synonyms  # comma separated list of synonyms
+  t.string     :name,     null: false
+  t.string     :key,      null: false
+  t.references :place,    null: false
+  t.string     :alt_names  # comma separated list of alternate names (synonyms)
 
   ## timestamp at last
   t.timestamps
 end
 
-add_index :continents, :key,  :unique => true
+add_index :continents, :key, unique: true
 
 
 create_table :countries do |t|
-  t.string  :title,  :null => false
-  t.string  :key,    :null => false
-  t.string  :code,   :null => false  # short three letter code (FIFA country code e.g. ITA)
-  t.string  :synonyms  # comma separated list of synonyms
-  t.integer :pop,    :null => false    # population count
-  t.integer :area,   :null => false    #  area in square km (sq. km)
+  t.string     :name,   null: false
+  t.string     :key,    null: false
+  t.references :place,  null: false
+  t.string     :code,   null: false  # short three letter code (FIFA country code e.g. ITA)
+  t.string     :alt_names  # comma separated list of alternate names (synonyms)
+  t.integer    :pop,    null: false    # population count
+  t.integer    :area,   null: false    #  area in square km (sq. km)
   t.references :continent
   t.references :country    # for supra(nationals) n depend(encies)
-  
+
   ## flags (use single int named flags - why? why not?  
-  t.boolean :s,  :null => false, :default => false   # supra(national) flag e.g. eu
-  t.boolean :c,  :null => false, :default => false   # country flag (is this needed?)
-  t.boolean :d,  :null => false, :default => false   # dependency flag
- 
+  t.boolean :s,  null: false, default: false   # supra(national) flag e.g. eu
+  t.boolean :c,  null: false, default: false   # country flag (is this needed?)
+  t.boolean :d,  null: false, default: false   # dependency flag
+
   # extras
   t.string  :motor      # optional auto motor (vehicle) licene plate
   t.string  :iso2       # optional iso two letter country code
@@ -45,8 +102,8 @@ create_table :countries do |t|
 end
 
 
-add_index :countries, :key,  :unique => true
-add_index :countries, :code, :unique => true
+add_index :countries, :key,  unique: true
+add_index :countries, :code, unique: true
 
 
 ######
@@ -54,88 +111,66 @@ add_index :countries, :code, :unique => true
 #
 # used for state/provice/land/regioni/etc.
 create_table :regions do |t|
-  t.string :title,  :null => false
-  t.string :key,    :null => false
+  t.string     :name,   null: false
+  t.string     :key,    null: false
+  t.references :place,  null: false
   t.string :code     # short two or three letter code e.g. NY, OAX, etc.
   t.string :abbr     # optional conventional abbrevation (e.g. Stmk., Gto., etc.)
   t.string :iso      # iso code
   t.string :nuts     # nuts code (europe/eu only)
-  t.string :synonyms  # comma separated list of synonyms
-  t.references :country, :null => false
+  t.string     :alt_names  # comma separated list of alternate names (synonyms)
+  t.references :country,  null: false
   t.integer :pop     # optional population count
   t.integer :area    # optional area in square km (sq. km)
   t.timestamps
 end
 
-add_index :regions, [:key, :country_id], :unique => true
+add_index :regions, [:key, :country_id], unique: true
 
 
 create_table :cities do |t|
-  t.string :title, :null => false
-  t.string :key,   :null => false
-  t.string :code     # short three letter code (ITAT/airport code e.g. NYC or VIE)
-  t.string :synonyms  # comma separated list of synonyms
-  t.references :country,  :null => false
+  t.string     :name,   null: false
+  t.string     :key,    null: false
+  t.references :place,  null: false
+  t.string     :code     # short three letter code (ITAT/airport code e.g. NYC or VIE)
+  t.string     :alt_names  # comma separated list of alternate names (synonyms)
+  t.references :country,  null: false
   t.references :region    # optional for now
   t.references :city  # optional parent (e.g. metro for city, or city for district)
   t.integer :pop     # optional population count (city proper)
   t.integer :popm    # optional population count (metropolitan/aglomeration)
   t.integer :area    # optional area in square km (sq. km)
-  t.float   :lat   # optional for now
-  t.float   :lng   # optional for now
+
+  ## t.float   :lat   # optional for now  --  FIX: remove?? moved to places
+  ## t.float   :lng   # optional for now  --  FIX: remove?? moved to places
 
   ## flags (use single int named flags - why? why not?    
-  t.boolean :m,  :null => false, :default => false   # metro flag
-  t.boolean :c,  :null => false, :default => false   # city flag (is this needed?)
-  t.boolean :d,  :null => false, :default => false   # district flag
+  t.boolean :m,  null: false, default: false   # metro flag
+  t.boolean :c,  null: false, default: false   # city flag (is this needed?)
+  t.boolean :d,  null: false, default: false   # district flag
 
-  ### t.boolean :capital, :null => false, :default => false  # is national captial?
-  
-  t.timestamps
-end
+  ### t.boolean :capital, null: false, default: false  # is national captial?
 
-create_table :tags do |t|
-  t.string :key,  :null => false
-  t.string :slug, :null => false
-  t.string :title    # todo: make required?
-  t.integer :grade, :null => false, :default => 1  # grade/tier e.g. 1/2/3 for now
-  ## todo: add parent or similar for hierachy (for tag stacks/packs)
   t.timestamps
 end
 
-add_index :tags, :key,  :unique => true
-
-create_table :taggings do |t|
-  t.references :tag,  :null => false
-  t.references :taggable, :polymorphic => true
-  t.timestamps   # todo: use only t.datetime :created_at (do we get ar magic? is updated used/needed??)
-end
-
-add_index :taggings, :tag_id
-add_index :taggings, [:taggable_id, :taggable_type]
-
 
 create_table :langs do |t|  # langs == languages (e.g. en/English, de/Deutsch, etc.)
-  t.string :key,   :null => false
-  t.string :title, :null => false
+  t.string :key,   null: false
+  t.string :name,  null: false
   t.timestamps
 end
 
+
 create_table :usages do |t|  # join table for countries_langs
-  t.references :country,  :null => false
-  t.references :lang,     :null => false
-  t.boolean    :official, :null => false, :default => true  # is_official language in country
-  t.boolean    :minor,    :null => false, :default => false # spoken by minority 
+  t.references :country,  null: false
+  t.references :lang,     null: false
+  t.boolean    :official, null: false, default: true  # is_official language in country
+  t.boolean    :minor,    null: false, default: false # spoken by minority 
   t.float      :percent     # usage in percent e.g. 90.0, 0.55, etc.
   t.timestamps
 end
 
-### fix: move to PropDb ? into props gem? why? why not?
-create_table :props do |t|
-  t.string :key,   :null => false
-  t.string :value, :null => false
-  t.timestamps
-end
 
 end # method up
 

data/lib/worlddb/stats.rb

@@ -13,12 +13,18 @@ module WorldDb
       puts " #{'%5d' % City.where(m: true).where(c: false).count} metros"
       puts " #{'%5d' % City.where(c: true).count} cities (#{City.where(c: true).where(m: true).count} metros)"
       puts " #{'%5d' % City.where(d: true).count} districts"
-      puts " #{'%5d' % Tag.count} tags"
-      puts " #{'%5d' % Tagging.count} taggings"
+#      puts " #{'%5d' % Tag.count} tags"
+#      puts " #{'%5d' % Tagging.count} taggings"
+      puts " #{'%5d' % Place.count} places"
+      puts " #{'%5d' % Name.count} names"
       puts " #{'%5d' % Lang.count} langs"
       puts " #{'%5d' % Usage.count} usages"
     end
-    
+
+
+#
+# fix: move to ConfDb for (re)use !!!!!
+
     def props
       puts "Props:"
       Prop.order( 'created_at asc' ).all.each do |prop|

data/lib/worlddb/utils.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 
+######
+# fix:
+#  move to textutils ???
 
 class Time
   

data/lib/worlddb/version.rb

@@ -1,9 +1,4 @@
 
 module WorldDb
-  VERSION = '1.8.2'  # sync version w/ sport.db - why? why not?
+  VERSION = '2.0.0'  # sync version w/ sport.db - why? why not?
 end
-
-###########################################
-## fix: remove old alias for WorldDb
-
-WorldDB = WorldDb