worlddb 0.6.0 → 0.6.1

data/db/america/ca/regions.txt

@@ -21,7 +21,7 @@ pe, Prince Edward Island,        5686, eastern canada|atlantic canada|maritimes
 nl, Newfoundland and Labrador, 370511, eastern canada|atlantic canada
 
 on, Ontario,                   908608, eastern canada|central canada
-qc, Québec                    1356547, eastern canada|central canada
+qc, Québec,                   1356547, eastern canada|central canada
 
 nt, Northwest Territories,    1143793, northern canada|territories
 yt, Yukon,                     474713, northern canada|territories

data/db/america/countries.txt

@@ -22,7 +22,7 @@ sv, El Salvador,   SLV
   
 ar, Argentina,     ARG, 2780400,  40518425, south america|es
 bo, Bolivia,       BOL, 1098581,  10907778, south america|es
-br, Brazil,        BRA, 8514215, 192380000, south america|pt-br
+br, Brazil,        BRA, 8514215, 192380000, south america|pt
 cl, Chile,         CHI,  755696,  16763470, south america|es
 co, Colombia,      COL, 1138748,  46413791, south america|es
 ec, Ecuador,       ECU,  258238,  15007343, south america|es

data/db/europe/countries.txt

@@ -70,7 +70,7 @@ ad, Andorra,     AND,         468,      85_082
 al, Albania,     ALB,      28_748,   3_002_859
 by, Belarus,     BLR,     207_600,   9_643_566
 ch, Switzerland, SUI,      41_285,   7_866_500,   schengen|uefa|un    # NB: ISO (CHE) <> FIFA (SUI)
-hr, Croatia,     CRO,       56_594,   4_290_612     # NB: ISO (HRV) <> FIFA (CRO);  local name: Hrvatska
+hr, Croatia,     CRO,      56_594,   4_290_612     # NB: ISO (HRV) <> FIFA (CRO);  local name: Hrvatska
 rs, Serbia,      SRB,      88_361,   7_276_604
 ru, Russia,       RUS, 17_098_242, 142_517_670,   asia|g8|g20
 tr, Turkey,       TUR,    783_562,  79_749_461,   asia

data/db/europe/fr/cities.txt

@@ -1,7 +1,7 @@
 # top 10 cities
 
 paris,        Paris,       region:if, 2_234_105, m:10_755_000
-marseille,    Marseille    region:ac,   850_602, m: 1_582_000
+marseille,    Marseille,   region:ac,   850_602, m: 1_582_000
 lyon,         Lyon,        region:ra,   479_803, m: 1_542_000
 toulouse,     Toulouse,    region:mp,   440_204, m:   880_000
 nice,         Nice,        region:ac,   340_735, m:   962_000

data/lib/worlddb/models/region.rb

@@ -10,6 +10,7 @@ class Region < ActiveRecord::Base
   has_many :tags,     :through => :taggings
 
   validates :key, :format => { :with => /^[a-z]{2,}$/, :message => 'expected two or more lowercase letters a-z' }
+  validates :code, :format => { :with => /^[A-Z_]{2,3}$/, :message => 'expected two or three uppercase letters A-Z (and _)' }, :allow_nil => true
 
   def self.create_from_ary!( regions, more_values={} )
     regions.each do |values|

data/lib/worlddb/reader.rb

@@ -140,6 +140,11 @@ private
     reader.each_line do |attribs, values|
   
       value_numbers = []
+      
+      ### todo: remove attribs[:tags] from attribs? lets us pass in "default" tags - why? why not?
+      ##
+      
+      value_tags = []
 
       if clazz == City
         attribs[ :c ] = true   # assume city type by default (use metro,district to change in fixture)
@@ -151,7 +156,8 @@ private
       values.each_with_index do |value,index|
         if value =~ /^region:/   ## region:
           value_region_key = value[7..-1]  ## cut off region: prefix
-          value_region = Region.find_by_key!( value_region_key )
+          ## NB: requires country_id to make unique!
+          value_region = Region.find_by_key_and_country_id!( value_region_key, attribs[:country_id] )
           attribs[ :region_id ] = value_region.id
         elsif value =~ /^metro$/   ## metro(politan area)
           attribs[ :c ] = false   # turn off default c|city flag; make it m|metro only
@@ -184,15 +190,35 @@ private
           value_popm = value_popm_str.gsub(/[ _]/, '').to_i
           attribs[ :popm ] = value_popm
           attribs[ :m ] = true   #  auto-mark city as m|metro too
-        elsif value =~ /^[A-Z]{3}$/  ## assume three-letter code
+        elsif value =~ /^[A-Z]{2,3}$/  ## assume two or three-letter code
           attribs[ :code ] = value
         elsif value =~ /(^[0-9]{1,2}$)|(^[0-9][0-9 _]+[0-9]$)/    ## numeric (nb: can use any _ or spaces inside digits e.g. 1_000_000 or 1 000 000)
           value_numbers << value.gsub(/[ _]/, '').to_i
-        elsif (values.size==(index+3)) && value =~ /^[a-z0-9\|_ ]+$/   # tags must be last entry
-          puts "   skipping tags: #{value}"
+        elsif (values.size==(index+1)) && value =~ /^[a-z0-9\|_ ]+$/   # tags must be last entry
+
+          puts "   processing tags: >>#{value}<<..."
+ 
+          tag_keys = value.split('|')
+  
+          ## unify; replace _w/ space; remove leading n trailing whitespace
+          tag_keys = tag_keys.map do |key|
+            key = key.gsub( '_', ' ' )
+            key = key.strip
+            key
+          end
+          
+          tag_keys.each do |key|
+            tag = Tag.find_by_key( key )
+            if tag.nil?  # create tag if it doesn't exit
+              puts "   creating tag >#{key}<"
+              tag = Tag.create!( key: key )
+            end
+            value_tags << tag
+          end
+    
         else
           # issue warning: unknown type for value
-          puts "*** warning: unknown type for value >#{value}<"
+          puts "!!!! >>>> warning: unknown type for value >#{value}<"
         end
       end
       
@@ -206,7 +232,15 @@ private
         end
       end
 
-      rec = clazz.find_by_key( attribs[ :key ] )
+      rec = nil
+      
+      if clazz == Region  ## requires country_id
+        ## todo: assert that country_id is present/valid, that is, NOT null
+        rec = clazz.find_by_key_and_country_id( attribs[ :key ], attribs[ :country_id] )
+      else
+        rec = clazz.find_by_key( attribs[ :key ] )
+      end
+
       if rec.present?
         ## nb: [17..-1] cut off WorldDB::Models:: in name
         puts "*** update #{clazz.name[17..-1].downcase} #{rec.id}-#{rec.key}:"
@@ -218,6 +252,14 @@ private
       puts attribs.to_json
    
       rec.update_attributes!( attribs )
+      
+      ## add tags
+      
+      value_tags.each do |tag|
+        rec.tags << tag
+      end
+      
+      ### fix/todo: check tag_ids and only update diff (add/remove ids)
         
     end # each_line
             

data/lib/worlddb/schema.rb

@@ -27,10 +27,14 @@ create_table :countries do |t|
   t.string  :motor   # optional auto motor (vehicle) licene plate
 end
 
+add_index :countries, :key,  :unique => true
+add_index :countries, :code, :unique => true
+
+
 create_table :regions do |t|
   t.string :title,  :null => false
   t.string :key,    :null => false
-  t.string :code     # short three letter code
+  t.string :code     # short two or three letter code e.g. NY, OAX, etc.
   t.string :synonyms  # comma separated list of synonyms
   t.references :country, :null => false
   t.integer :pop     # optional population count
@@ -38,6 +42,9 @@ create_table :regions do |t|
   t.timestamps
 end
 
+add_index :regions, [:key, :country_id], :unique => true
+
+
 create_table :cities do |t|
   t.string :title, :null => false
   t.string :key,   :null => false
@@ -66,6 +73,8 @@ create_table :tags do |t|
   t.timestamps
 end
 
+add_index :tags, :key,  :unique => true
+
 create_table :taggings do |t|
   t.references :tag,  :null => false
   t.references :taggable, :polymorphic => true

data/lib/worlddb/version.rb

@@ -1,5 +1,5 @@
 
 module WorldDB
-  VERSION = '0.6.0'
+  VERSION = '0.6.1'
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: worlddb
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: 
   segments: 
   - 0
   - 6
-  - 0
-  version: 0.6.0
+  - 1
+  version: 0.6.1
 platform: ruby
 authors: 
 - Gerald Bauer