worlddb 1.6.4 → 1.6.5

data/Manifest.txt

@@ -23,3 +23,5 @@ lib/worlddb/schema.rb
 lib/worlddb/stats.rb
 lib/worlddb/utils.rb
 lib/worlddb/version.rb
+test/helper.rb
+test/test_values.rb

data/lib/worlddb/models/city.rb

@@ -3,6 +3,14 @@
 module WorldDb::Models
 
 class City < ActiveRecord::Base
+  
+  extend WorldDb::TagHelper  # will add self.find_tags, self.find_tags_in_hash!, etc.
+
+  # NB: use extend - is_<type>? become class methods e.g. self.is_<type>? for use in
+  #   self.create_or_update_from_values
+  extend TextUtils::ValueHelper  # e.g. is_year?, is_region?, is_address?, is_taglist? etc.
+
+  
   self.table_name = 'cities'
 
   belongs_to :country, :class_name => 'Country', :foreign_key => 'country_id'
@@ -109,6 +117,152 @@ class City < ActiveRecord::Base
     end # each city
   end
 
+
+  def self.create_or_update_from_titles( titles, more_attributes = {} )  # ary of titles e.g. ['Wien', 'Graz'] etc.
+
+    # fix: add/configure logger for ActiveRecord!!!
+    logger = LogKernel::Logger.root
+
+    titles.each do |title|
+
+      new_attributes = {}
+      key = TextUtils.title_to_key( title )   # auto generate key from title
+
+      # check if it exists
+      # todo/fix: add country_id for lookup?
+      city = City.find_by_key( key )
+      if city.present?
+        logger.debug "update city #{city.id}-#{city.key}:"
+      else
+        logger.debug "create city:"
+        city = City.new
+        new_attributes[ :key ] = key
+      end
+        
+      new_attributes[ :title ] = title
+     
+      ## merge in passed in attribes (e.g. country_id etc.)
+      new_attributes.merge!( more_attributes )
+      
+      logger.debug new_attributes.to_json
+   
+      city.update_attributes!( new_attributes )
+      
+      ### todo/fix: add captial ref to country/region
+    end # each city
+  end  # method create_or_update_from_titles
+
+
+  def self.create_or_update_from_values( new_attributes, values, opts={} )
+
+    ## opts e.g. :skip_tags true|false
+
+    ## fix: add/configure logger for ActiveRecord!!!
+    logger = LogKernel::Logger.root
+
+    value_numbers     = []
+    value_tag_keys    = []
+      
+    ### check for "default" tags - that is, if present new_attributes[:tags] remove from hash
+    value_tag_keys += find_tags_in_hash!( new_attributes )
+
+    new_attributes[ :c ] = true   # assume city type by default (use metro,district to change in fixture)
+
+    ## check for optional values
+    values.each_with_index do |value,index|
+      if value =~ /^region:/   ## region:
+        value_region_key = value[7..-1]  ## cut off region: prefix
+        ## NB: requires country_id to make unique!
+        value_region = Region.find_by_key_and_country_id!( value_region_key, new_attributes[:country_id] )
+        new_attributes[ :region_id ] = value_region.id
+      elsif value =~ /^metro$/   ## metro(politan area)
+        new_attributes[ :c ] = false   # turn off default c|city flag; make it m|metro only
+        new_attributes[ :m ] = true    
+      elsif value =~ /^country:/   ## country:
+        value_country_key = value[8..-1]  ## cut off country: prefix
+        value_country = Country.find_by_key!( value_country_key )
+        new_attributes[ :country_id ] = value_country.id
+      elsif value =~ /^metro:/   ## metro:
+        value_city_key = value[6..-1]  ## cut off metro: prefix
+        value_city = City.find_by_key!( value_city_key )
+        new_attributes[ :city_id ] = value_city.id
+      elsif value =~ /^city:/   ## city:
+        value_city_key = value[5..-1]  ## cut off city: prefix
+        value_city = City.find_by_key!( value_city_key )
+        new_attributes[ :city_id ] = value_city.id
+        new_attributes[ :c ] = false # turn off default c|city flag; make it d|district only
+        new_attributes[ :d ] = true  
+      elsif value =~ /^m:/   ## m:
+        value_popm_str = value[2..-1]  ## cut off m: prefix
+        value_popm = value_popm_str.gsub(/[ _]/, '').to_i
+        new_attributes[ :popm ] = value_popm
+        new_attributes[ :m ] = true   #  auto-mark city as m|metro too
+      elsif is_region?( value )  ## assume region code e.g. TX for city
+        value_region = Region.find_by_key_and_country_id!( value.downcase, new_attributes[:country_id] )
+        new_attributes[ :region_id ] = value_region.id
+      elsif value =~ /^[A-Z]{2,3}$/  ## assume two or three-letter code
+        new_attributes[ :code ] = value
+      elsif value =~ /^([0-9][0-9 _]+[0-9]|[0-9]{1,2})(?:\s*(?:km2|km²)\s*)$/
+        ## allow numbers like 453 km²
+        value_numbers << value.gsub( 'km2', '').gsub( 'km²', '' ).gsub(/[ _]/, '').to_i
+      elsif value =~ /^([0-9][0-9 _]+[0-9])|([0-9]{1,2})$/    ## numeric (nb: can use any _ or spaces inside digits e.g. 1_000_000 or 1 000 000)
+        value_numbers << value.gsub(/[ _]/, '').to_i
+      elsif (values.size==(index+1)) && is_taglist?( value )   # tags must be last entry
+        logger.debug "   found tags: >>#{value}<<"
+        value_tag_keys += find_tags( value )
+      else
+        # issue warning: unknown type for value
+        logger.warn "unknown type for value >#{value}<"
+      end
+    end # each value
+
+    if value_numbers.size > 0
+      new_attributes[ :pop  ] = value_numbers[0]   # assume first number is pop for cities
+      new_attributes[ :area ] = value_numbers[1]  
+    end  # if value_numbers.size > 0
+
+    rec = City.find_by_key( new_attributes[ :key ] )
+
+    if rec.present?
+      logger.debug "update City #{rec.id}-#{rec.key}:"
+    else
+      logger.debug "create City:"
+      rec = City.new
+    end
+
+    logger.debug new_attributes.to_json
+
+    rec.update_attributes!( new_attributes )
+
+      ##################
+      ## add taggings
+
+      ## todo/fix: reuse - move add taggings into method etc.
+
+      if value_tag_keys.size > 0
+
+        if opts[:skip_tags].present?
+          logger.debug "   skipping add taggings (flag skip_tag)"
+        else
+          value_tag_keys.uniq!  # remove duplicates
+          logger.debug "   adding #{value_tag_keys.size} taggings: >>#{value_tag_keys.join('|')}<<..."
+
+          ### fix/todo: check tag_ids and only update diff (add/remove ids)
+
+          value_tag_keys.each do |key|
+            tag = Tag.find_by_key( key )
+            if tag.nil?  # create tag if it doesn't exit
+              logger.debug "   creating tag >#{key}<"
+              tag = Tag.create!( key: key )
+            end
+            rec.tags << tag
+          end
+        end
+      end
+
+    rec
+  end # method create_or_update_from_values
+
 end # class Cities
 
 end # module WorldDb::Models

data/lib/worlddb/models/country.rb

@@ -3,6 +3,14 @@
 module WorldDb::Models
 
 class Country < ActiveRecord::Base
+
+  extend WorldDb::TagHelper  # will add self.find_tags, self.find_tags_in_hash!, etc.
+
+  # NB: use extend - is_<type>? become class methods e.g. self.is_<type>? for use in
+  #   self.create_or_update_from_values
+  extend TextUtils::ValueHelper  # e.g. is_year?, is_region?, is_address?, is_taglist? etc.
+
+
   self.table_name = 'countries'
 
   belongs_to :continent, :class_name => 'Continent', :foreign_key => 'continent_id'
@@ -95,6 +103,153 @@ class Country < ActiveRecord::Base
     end # each country
   end
 
+
+  def self.create_or_update_from_values( new_attributes, values, opts={} )
+
+    ## opts e.g. :skip_tags true|false
+
+    ## fix: add/configure logger for ActiveRecord!!!
+    logger = LogKernel::Logger.root
+
+      value_numbers     = []
+      value_tag_keys    = []
+      value_cities      = []
+
+      ### check for "default" tags - that is, if present new_attributes[:tags] remove from hash
+      value_tag_keys += find_tags_in_hash!( new_attributes )
+
+
+      new_attributes[ :c ] = true   # assume country type by default (use supra,depend to change)
+      
+      ## check for optional values
+      values.each_with_index do |value,index|
+        if value =~ /^supra$/   ## supra(national)
+          new_attributes[ :c ] = false   # turn off default c|country flag; make it s|supra only
+          new_attributes[ :s ] = true
+          ## auto-add tag supra
+          value_tag_keys << 'supra'
+        elsif value =~ /^supra:/   ## supra:
+          value_country_key = value[6..-1]  ## cut off supra: prefix
+          value_country = Country.find_by_key!( value_country_key )
+          new_attributes[ :country_id ] = value_country.id
+        elsif value =~ /^country:/   ## country:
+          value_country_key = value[8..-1]  ## cut off country: prefix
+          value_country = Country.find_by_key!( value_country_key )
+          new_attributes[ :country_id ] = value_country.id
+          new_attributes[ :c ] = false # turn off default c|country flag; make it d|depend only
+          new_attributes[ :d ] = true
+          ## auto-add tag supra
+          value_tag_keys << 'territory'  # rename tag to dependency? why? why not?
+        elsif value =~ /^[A-Z]{2,3}$/  ## assume two or three-letter code
+          new_attributes[ :code ] = value
+        elsif value =~ /^([0-9][0-9 _]+[0-9]|[0-9]{1,2})(?:\s*(?:km2|km²)\s*)$/
+          ## allow numbers like 453 km²
+          value_numbers << value.gsub( 'km2', '').gsub( 'km²', '' ).gsub(/[ _]/, '').to_i
+        elsif value =~ /^([0-9][0-9 _]+[0-9])|([0-9]{1,2})$/    ## numeric (nb: can use any _ or spaces inside digits e.g. 1_000_000 or 1 000 000)
+          value_numbers << value.gsub(/[ _]/, '').to_i
+        elsif (values.size==(index+1)) && is_taglist?( value )   # tags must be last entry
+          logger.debug "   found tags: >>#{value}<<"
+          value_tag_keys += find_tags( value )
+        else
+          
+          ### assume it is the capital city - mark it for auto add
+          value_cities << value
+          next
+
+          # issue warning: unknown type for value
+          # logger.warn "unknown type for value >#{value}<"
+        end
+      end # each value
+
+      if value_numbers.size > 0
+          new_attributes[ :area ] = value_numbers[0]
+          new_attributes[ :pop  ] = value_numbers[1]
+      end
+
+=begin
+            # auto-add tags
+            area = value_numbers[0]
+            pop  = value_numbers[1]
+            
+            # categorize into brackets
+            if area >= 1_000_000
+              value_tag_keys << 'area_1_000_000_n_up'
+            elsif area >= 100_000
+              value_tag_keys << 'area_100_000_to_1_000_000'
+            elsif area >= 1000
+              value_tag_keys << 'area_1_000_to_100_000'
+            else
+              value_tag_keys << 'area_1_000_n_less' # microstate
+            end
+
+            # include all
+            value_tag_keys << 'area_100_000_n_up'  if area >= 100_000
+            value_tag_keys << 'area_1_000_n_up'    if area >=   1_000
+
+            
+            # categorize into brackets
+            if pop >= 100_000_000
+              value_tag_keys << 'pop_100m_n_up'
+            elsif pop >= 10_000_000
+              value_tag_keys << 'pop_10m_to_100m'
+            elsif pop >= 1_000_000
+              value_tag_keys << 'pop_1m_to_10m'
+            else
+              value_tag_keys << 'pop_1m_n_less'
+            end
+            
+            # include all
+            value_tag_keys << 'pop_10m_n_up'  if pop >= 10_000_000
+            value_tag_keys << 'pop_1m_n_up'   if pop >=  1_000_000
+=end
+
+      rec = Country.find_by_key( new_attributes[ :key ] )
+
+      if rec.present?
+        logger.debug "update Country #{rec.id}-#{rec.key}:"
+      else
+        logger.debug "create Country:"
+        rec = Country.new
+      end
+      
+      logger.debug new_attributes.to_json
+   
+      rec.update_attributes!( new_attributes )
+
+      #################
+      ## auto add capital cities
+
+      City.create_or_update_from_titles( value_cities, country_id: rec.id )
+
+      ##################
+      ## add taggings 
+
+      if value_tag_keys.size > 0
+        
+        if opts[:skip_tags].present?
+          logger.debug "   skipping add taggings (flag skip_tag)"
+        else
+          value_tag_keys.uniq!  # remove duplicates
+          logger.debug "   adding #{value_tag_keys.size} taggings: >>#{value_tag_keys.join('|')}<<..."
+
+          ### fix/todo: check tag_ids and only update diff (add/remove ids)
+
+          value_tag_keys.each do |key|
+            tag = Tag.find_by_key( key )
+            if tag.nil?  # create tag if it doesn't exit
+              logger.debug "   creating tag >#{key}<"
+              tag = Tag.create!( key: key )
+            end
+            rec.tags << tag
+          end
+        end
+      end
+
+    rec
+
+  end # method create_or_update_from_values
+
+
 end # class Country
 
 

data/lib/worlddb/models/region.rb

@@ -1,7 +1,16 @@
+# encoding: UTF-8
+
 module WorldDb::Models
 
 class Region < ActiveRecord::Base
 
+  extend WorldDb::TagHelper  # will add self.find_tags, self.find_tags_in_hash!, etc.
+
+  # NB: use extend - is_<type>? become class methods e.g. self.is_<type>? for use in
+  #   self.create_or_update_from_values
+  extend TextUtils::ValueHelper  # e.g. is_year?, is_region?, is_address?, is_taglist? etc.
+
+
   belongs_to :country, :class_name => 'Country', :foreign_key => 'country_id'
 
   has_many :cities, :class_name => 'City', :foreign_key => 'region_id'
@@ -9,7 +18,7 @@ class Region < ActiveRecord::Base
   has_many :taggings, :as => :taggable
   has_many :tags,     :through => :taggings
 
-  validates :key, :format => { :with => /^[a-z]{2,}$/, :message => 'expected two or more lowercase letters a-z' }
+  validates :key, :format => { :with => /^[a-z]+$/, :message => 'expected one or more lowercase letters a-z' }
   validates :code, :format => { :with => /^[A-Z_]{2,3}$/, :message => 'expected two or three uppercase letters A-Z (and _)' }, :allow_nil => true
 
 
@@ -48,6 +57,109 @@ class Region < ActiveRecord::Base
     end # each region
   end
 
+
+  def self.create_or_update_from_values( new_attributes, values, opts={} )
+
+    ## opts e.g. :skip_tags true|false
+
+    ## fix: add/configure logger for ActiveRecord!!!
+    logger = LogKernel::Logger.root
+
+    value_numbers     = []
+    value_tag_keys    = []
+    value_cities      = []
+
+    ### check for "default" tags - that is, if present new_attributes[:tags] remove from hash
+    value_tag_keys += find_tags_in_hash!( new_attributes )
+
+    ## check for optional values
+    values.each_with_index do |value,index|
+      if value =~ /^country:/   ## country:
+        value_country_key = value[8..-1]  ## cut off country: prefix
+        value_country = Country.find_by_key!( value_country_key )
+        new_attributes[ :country_id ] = value_country.id
+      elsif value =~ /^[A-Z]{2,3}$/  ## assume two or three-letter code
+        new_attributes[ :code ] = value
+      elsif value =~ /^([0-9][0-9 _]+[0-9]|[0-9]{1,2})(?:\s*(?:km2|km²)\s*)$/
+        ## allow numbers like 453 km²
+        value_numbers << value.gsub( 'km2', '').gsub( 'km²', '' ).gsub(/[ _]/, '').to_i
+      elsif value =~ /^([0-9][0-9 _]+[0-9])|([0-9]{1,2})$/    ## numeric (nb: can use any _ or spaces inside digits e.g. 1_000_000 or 1 000 000)
+        value_numbers << value.gsub(/[ _]/, '').to_i
+      elsif (values.size==(index+1)) && is_taglist?( value )   # tags must be last entry
+        logger.debug "   found tags: >>#{value}<<"
+        value_tag_keys += find_tags( value )
+      else
+          
+        ### assume it is the capital city - mark it for auto add
+        value_cities << value
+        next
+   
+        # issue warning: unknown type for value
+        # logger.warn "unknown type for value >#{value}<"
+      end
+    end # each value
+
+    if value_numbers.size > 0
+      new_attributes[ :area ] = value_numbers[0]
+      new_attributes[ :pop  ] = value_numbers[1]
+    end  # if value_numbers.size > 0
+
+    ## todo: assert that country_id is present/valid, that is, NOT null
+    rec = Region.find_by_key_and_country_id( new_attributes[ :key ], new_attributes[ :country_id] )
+
+    if rec.present?
+      logger.debug "update Region #{rec.id}-#{rec.key}:"
+    else
+      logger.debug "create Region:"
+      rec = Region.new
+    end
+      
+    logger.debug new_attributes.to_json
+   
+    rec.update_attributes!( new_attributes )
+
+    #################
+    # auto add capital cities
+
+    City.create_or_update_from_titles( value_cities,
+                                            region_id:  rec.id,
+                                            country_id: rec.country_id )
+
+    ### todo/fix: add captial ref to country/region
+
+
+    ##################
+    # add taggings 
+
+    ## todo/fix: reuse - move add taggings into method etc.
+
+      if value_tag_keys.size > 0
+        
+        if opts[:skip_tags].present?
+          logger.debug "   skipping add taggings (flag skip_tag)"
+        else
+          value_tag_keys.uniq!  # remove duplicates
+          logger.debug "   adding #{value_tag_keys.size} taggings: >>#{value_tag_keys.join('|')}<<..."
+
+          ### fix/todo: check tag_ids and only update diff (add/remove ids)
+
+          value_tag_keys.each do |key|
+            tag = Tag.find_by_key( key )
+            if tag.nil?  # create tag if it doesn't exit
+              logger.debug "   creating tag >#{key}<"
+              tag = Tag.create!( key: key )
+            end
+            rec.tags << tag
+          end
+        end
+      end
+
+    rec
+
+  end # method create_or_update_from_values
+
+
+
 end # class Region
 
 end # module WorldDb::Models

data/lib/worlddb/reader.rb

@@ -2,6 +2,40 @@
 
 module WorldDb
 
+
+  ### fix: move to textutils
+  ##   PlusReaderWrapper find a better name than Plus?
+  #
+  #  todo: also add a ValuesReaderPlus and use it
+
+  class HashReaderPlus
+    include LogUtils::Logging
+
+    def initialize( name, include_path )
+       @name          = name
+       @include_path  = include_path
+    end
+
+    attr_reader :name
+    attr_reader :include_path
+
+    def each
+      path          = "#{include_path}/#{name}.yml"
+      reader        = HashReader.new( path )
+
+      logger.info "parsing data '#{name}' (#{path})..."
+
+      reader.each do |key, value|
+        yield( key, value )
+      end
+      Prop.create_from_fixture!( name, path )
+    end
+
+  end # class HashReaderPlus
+
+
+
+
 class Reader
 
   include LogUtils::Logging
@@ -114,22 +148,16 @@ class Reader
 
 
   def load_regions_xxx( country_key, xxx, name )
-    path = "#{include_path}/#{name}.yml"
-
-    logger.info "parsing data '#{name}' (#{path})..."
-
     country = Country.find_by_key!( country_key )
     logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
 
-    reader = HashReader.new( path )
+    reader = HashReaderPlus.new( name, include_path )
 
     reader.each do |key, value|
       region = Region.find_by_country_id_and_key!( country.id, key )
       region.send( "#{xxx}=", value )
       region.save!
     end
-
-    Prop.create_from_fixture!( name, path )
   end
 
 
@@ -141,27 +169,8 @@ class Reader
   end
 
 
-
-  def with_path_for( name )
-    ## todo: find a better name?
-    # e.g. find_path_for  or open_fixture_for ??
-
-    path = "#{include_path}/#{name}.yml"
-
-    logger.info "parsing data '#{name}' (#{path})..."
-    
-    yield( path )
-    
-    Prop.create_from_fixture!( name, path )
-  end
-
-
   def load_continent_refs( name )
-    path = "#{include_path}/#{name}.yml"
-
-    logger.info "parsing data '#{name}' (#{path})..."
-
-    reader = HashReader.new( path )
+    reader = HashReaderPlus.new( name, include_path )
 
     reader.each do |key, value|
       country = Country.find_by_key!( key )
@@ -169,10 +178,9 @@ class Reader
       country.continent_id = continent.id
       country.save!
     end
-
-    Prop.create_from_fixture!( name, path )
   end
 
+
   def load_continent_defs( name, more_values={} )
     path = "#{include_path}/#{name}.txt"
 
@@ -208,11 +216,9 @@ class Reader
 
   def load_langs( name )
     
-    with_path_for( name ) do |path|
-  
-      reader = HashReader.new( path )
+    reader = HashReaderPlus.new( name, include_path )
 
-      reader.each do |key, value|
+    reader.each do |key, value|
 
         logger.debug "adding lang >>#{key}<< >>#{value}<<..."
       
@@ -236,17 +242,14 @@ class Reader
         logger.debug lang_attribs.to_json
      
         lang.update_attributes!( lang_attribs )
-      end # each key,value
-    end # with_path_for
+    end # each key,value
 
   end # method load_langs
 
 
   def load_tags( name, more_values={} )
     
-    with_path_for( name ) do |path|
-
-      reader = HashReader.new( path )
+      reader = HashReaderPlus.new( name, include_path )
 
       grade = 1
     
@@ -291,19 +294,13 @@ class Reader
    
           tag.update_attributes!( tag_attribs )
         end
-      end # each key,value
-    
-    end # with_path_for
-    
+    end # each key,value
+
   end # method load_tags
 
 
   def load_usages( name )
-    path = "#{include_path}/#{name}.yml"
-
-    logger.info "parsing data '#{name}' (#{path})..."
-
-    reader = HashReader.new( path )
+    reader = HashReaderPlus.new( name, include_path )
 
     reader.each do |key, value|
       logger.debug "   adding langs >>#{value}<<to country >>#{key}<<"
@@ -322,28 +319,19 @@ class Reader
         Usage.create!( country_id: country.id, lang_id: lang.id, official: true, minor: false )
       end
     end
-
-    Prop.create_from_fixture!( name, path )
   end
 
 
   def load_xxx( xxx, name )
-    path = "#{include_path}/#{name}.yml"
-
-    logger.info "parsing data '#{name}' (#{path})..."
-
-    reader = HashReader.new( path )
+    reader = HashReaderPlus.new( name, include_path )
 
     reader.each do |key, value|
       country = Country.find_by_key!( key )
       country.send( "#{xxx}=", value )
       country.save!
     end
-
-    Prop.create_from_fixture!( name, path )
   end
 
-
 private
   def load_fixtures_for( clazz, name, more_values={} )  # load from file system
     path = "#{include_path}/#{name}.txt"
@@ -352,267 +340,13 @@ private
 
     reader = ValuesReader.new( path, more_values )
     
-    load_fixtures_worker_for( clazz, reader )
+    reader.each_line do |new_attributes, values|
+      opts = { skip_tags: skip_tags? }
+      clazz.create_or_update_from_values( new_attributes, values, opts )
+    end
 
     Prop.create_from_fixture!( name, path )
   end
-
-
-  def load_fixtures_worker_for( clazz, reader )
-   
-    ## NB: assumes active activerecord db connection
-    ##
-
-    reader.each_line do |attribs, values|
-  
-      value_numbers     = []
-      value_tag_keys    = []
-      value_cities      = []
-      
-      ### check for "default" tags - that is, if present attribs[:tags] remove from hash
-      
-      if attribs[:tags].present?
-        more_tag_keys = attribs[:tags].split('|')
-        attribs.delete(:tags)
-
-        ## unify; replace _w/ space; remove leading n trailing whitespace
-        more_tag_keys = more_tag_keys.map do |key|
-          key = key.gsub( '_', ' ' )
-          key = key.strip
-          key
-        end
-
-        value_tag_keys += more_tag_keys
-      end
-
-
-      if clazz == City
-        attribs[ :c ] = true   # assume city type by default (use metro,district to change in fixture)
-      elsif clazz == Country
-        attribs[ :c ] = true   # assume country type by default (use supra,depend to change)
-      end
-      
-      ## check for optional values
-      values.each_with_index do |value,index|
-        if value =~ /^region:/   ## region:
-          value_region_key = value[7..-1]  ## cut off region: prefix
-          ## NB: requires country_id to make unique!
-          value_region = Region.find_by_key_and_country_id!( value_region_key, attribs[:country_id] )
-          attribs[ :region_id ] = value_region.id
-        elsif value =~ /^metro$/   ## metro(politan area)
-          attribs[ :c ] = false   # turn off default c|city flag; make it m|metro only
-          attribs[ :m ] = true    
-        elsif value =~ /^supra$/   ## supra(national)
-          attribs[ :c ] = false   # turn off default c|country flag; make it s|supra only
-          attribs[ :s ] = true
-          ## auto-add tag supra
-          value_tag_keys << 'supra'
-        elsif value =~ /^supra:/   ## supra:
-          value_country_key = value[6..-1]  ## cut off supra: prefix
-          value_country = Country.find_by_key!( value_country_key )
-          attribs[ :country_id ] = value_country.id
-        elsif value =~ /^country:/   ## country:
-          value_country_key = value[8..-1]  ## cut off country: prefix
-          value_country = Country.find_by_key!( value_country_key )
-          attribs[ :country_id ] = value_country.id
-          attribs[ :c ] = false # turn off default c|country flag; make it d|depend only
-          attribs[ :d ] = true
-          ## auto-add tag supra
-          value_tag_keys << 'territory'  # rename tag to dependency? why? why not?
-        elsif value =~ /^metro:/   ## metro:
-          value_city_key = value[6..-1]  ## cut off metro: prefix
-          value_city = City.find_by_key!( value_city_key )
-          attribs[ :city_id ] = value_city.id
-        elsif value =~ /^city:/   ## city:
-          value_city_key = value[5..-1]  ## cut off city: prefix
-          value_city = City.find_by_key!( value_city_key )
-          attribs[ :city_id ] = value_city.id
-          attribs[ :c ] = false # turn off default c|city flag; make it d|district only
-          attribs[ :d ] = true  
-        elsif value =~ /^m:/   ## m:
-          value_popm_str = value[2..-1]  ## cut off m: prefix
-          value_popm = value_popm_str.gsub(/[ _]/, '').to_i
-          attribs[ :popm ] = value_popm
-          attribs[ :m ] = true   #  auto-mark city as m|metro too
-        elsif is_region?( value ) && clazz == City   ## assume region code e.g. TX for city
-          value_region = Region.find_by_key_and_country_id!( value.downcase, attribs[:country_id] )
-          attribs[ :region_id ] = value_region.id
-        elsif value =~ /^[A-Z]{2,3}$/  ## assume two or three-letter code
-          attribs[ :code ] = value
-        elsif value =~ /^([0-9][0-9 _]+[0-9]|[0-9]{1,2})(?:\s*(?:km2|km²)\s*)$/
-          ## allow numbers like 453 km²
-          value_numbers << value.gsub( 'km2', '').gsub( 'km²', '' ).gsub(/[ _]/, '').to_i
-        elsif value =~ /^([0-9][0-9 _]+[0-9])|([0-9]{1,2})$/    ## numeric (nb: can use any _ or spaces inside digits e.g. 1_000_000 or 1 000 000)
-          value_numbers << value.gsub(/[ _]/, '').to_i
-        elsif (values.size==(index+1)) && is_taglist?( value )   # tags must be last entry
-
-          logger.debug "   found tags: >>#{value}<<"
-
-          tag_keys = value.split('|')
-  
-          ## unify; replace _w/ space; remove leading n trailing whitespace
-          tag_keys = tag_keys.map do |key|
-            key = key.gsub( '_', ' ' )
-            key = key.strip
-            key
-          end
-          
-          value_tag_keys += tag_keys
-        else
-          
-          if clazz == Country || clazz == Region
-            ### assume it is the capital city - mark it for auto add
-            value_cities << value
-            next
-          end
-
-          # issue warning: unknown type for value
-          logger.warn "unknown type for value >#{value}<"
-        end
-      end # each value
-
-      
-      if value_numbers.size > 0
-        if clazz == City
-          attribs[ :pop  ] = value_numbers[0]   # assume first number is pop for cities
-          attribs[ :area ] = value_numbers[1]  
-        else   # countries,regions
-          attribs[ :area ] = value_numbers[0]
-          attribs[ :pop  ] = value_numbers[1]
-
-
-=begin
-          if clazz == Country
-            # auto-add tags
-            area = value_numbers[0]
-            pop  = value_numbers[1]
-            
-            # categorize into brackets
-            if area >= 1_000_000
-              value_tag_keys << 'area_1_000_000_n_up'
-            elsif area >= 100_000
-              value_tag_keys << 'area_100_000_to_1_000_000'
-            elsif area >= 1000
-              value_tag_keys << 'area_1_000_to_100_000'
-            else
-              value_tag_keys << 'area_1_000_n_less' # microstate
-            end
-
-            # include all
-            value_tag_keys << 'area_100_000_n_up'  if area >= 100_000
-            value_tag_keys << 'area_1_000_n_up'    if area >=   1_000
-
-            
-            # categorize into brackets
-            if pop >= 100_000_000
-              value_tag_keys << 'pop_100m_n_up'
-            elsif pop >= 10_000_000
-              value_tag_keys << 'pop_10m_to_100m'
-            elsif pop >= 1_000_000
-              value_tag_keys << 'pop_1m_to_10m'
-            else
-              value_tag_keys << 'pop_1m_n_less'
-            end
-            
-            # include all
-            value_tag_keys << 'pop_10m_n_up'  if pop >= 10_000_000
-            value_tag_keys << 'pop_1m_n_up'   if pop >=  1_000_000
-          end
-=end
-
-
-        end
-      end  # if value_numbers.size > 0
-
-      rec = nil
-      
-      if clazz == Region  ## requires country_id
-        ## todo: assert that country_id is present/valid, that is, NOT null
-        rec = clazz.find_by_key_and_country_id( attribs[ :key ], attribs[ :country_id] )
-      else
-        rec = clazz.find_by_key( attribs[ :key ] )
-      end
-
-      if rec.present?
-        ## nb: [17..-1] cut off WorldDB::Models:: in name
-        logger.debug "update #{clazz.name[17..-1].downcase} #{rec.id}-#{rec.key}:"
-      else
-        logger.debug "create #{clazz.name[17..-1].downcase}:"
-        rec = clazz.new
-      end
-      
-      logger.debug attribs.to_json
-   
-      rec.update_attributes!( attribs )
-
-      #################
-      ## auto add capital cities
-
-      value_cities.each do |city_title|
-        
-        city_attribs = {}
-        city_key = TextUtils.title_to_key( city_title )
-        
-        ## check if it exists
-        ## todo/fix: add country_id for lookup?
-        city = City.find_by_key( city_key )
-        if city.present?
-          logger.debug "update city #{city.id}-#{city.key}:"
-        else
-          logger.debug "create city:"
-          city = City.new
-          city_attribs[ :key ] = city_key
-        end
-        
-        city_attribs[ :title ] = city_title
-     
-        if clazz == Country
-          city_attribs[ :country_id ] = rec.id
-        elsif clazz == Region
-          city_attribs[ :region_id  ] = rec.id
-          city_attribs[ :country_id ] = rec.country_id
-        else
-          ## issue warning: unknown type for city!!!
-        end
-        
-        logger.debug city_attribs.to_json
-   
-        city.update_attributes!( city_attribs )
-        
-        ### todo/fix: add captial ref to country/region
-        
-       end # each city
-
-      
-      ##################
-      ## add taggings 
-
-      if value_tag_keys.size > 0
-        
-        if skip_tags?
-          logger.debug "   skipping add taggings (flag skip_tag)"
-        else
-          value_tag_keys.uniq!  # remove duplicates
-          logger.debug "   adding #{value_tag_keys.size} taggings: >>#{value_tag_keys.join('|')}<<..."
-
-          ### fix/todo: check tag_ids and only update diff (add/remove ids)
-
-          value_tag_keys.each do |key|
-            tag = Tag.find_by_key( key )
-            if tag.nil?  # create tag if it doesn't exit
-              logger.debug "   creating tag >#{key}<"
-              tag = Tag.create!( key: key )
-            end
-            rec.tags << tag
-          end
-        end
-      end
-
-        
-    end # each_line
-            
-  end # method load_fixture_worker_for
-
   
 end # class Reader
 end # module WorldDb

data/lib/worlddb/utils.rb

@@ -16,3 +16,39 @@ class Time
   end
   
 end # class Time
+
+
+##### fix/todo: move to helper folder - use one file per module/helper
+
+module WorldDb
+  module TagHelper
+
+  def find_tags( value )
+    # logger.debug "   found tags: >>#{value}<<"
+
+    tag_keys = value.split('|')
+
+    ## unify; replace _w/ space; remove leading n trailing whitespace
+    tag_keys = tag_keys.map do |key|
+      key = key.gsub( '_', ' ' )
+      key = key.strip
+      key
+    end
+
+    tag_keys # return tag keys as ary
+  end
+
+  def find_tags_in_hash!( h )
+    # NB: will remove :tags from hash
+
+    if h[:tags].present?
+      tag_keys = find_tags( h[:tags] )
+      h.delete(:tags)
+      tag_keys   # return tag keys as ary
+    else
+      []  # nothing found; return empty ary
+    end
+  end
+
+  end # module
+end # module WorldDb

data/lib/worlddb/version.rb

@@ -1,6 +1,6 @@
 
 module WorldDb
-  VERSION = '1.6.4'  # sync version w/ sport.db - why? why not?
+  VERSION = '1.6.5'  # sync version w/ sport.db - why? why not?
 end
 
 ###########################################

data/lib/worlddb.rb

@@ -26,6 +26,7 @@ require 'textutils'
 
 require 'worlddb/version'
 
+require 'worlddb/utils'
 require 'worlddb/models/prop'
 require 'worlddb/models/continent'
 require 'worlddb/models/country'
@@ -36,7 +37,6 @@ require 'worlddb/models/tagging'
 require 'worlddb/models/lang'
 require 'worlddb/models/usage'
 require 'worlddb/schema'       # NB: requires worlddb/models (include WorldDB::Models)
-require 'worlddb/utils'
 require 'worlddb/reader'
 require 'worlddb/deleter'
 require 'worlddb/stats'

data/test/helper.rb

@@ -0,0 +1,56 @@
+
+## $:.unshift(File.dirname(__FILE__))
+
+## minitest setup
+
+# require 'minitest/unit'
+require 'minitest/autorun'
+
+# include MiniTest::Unit  # lets us use TestCase instead of MiniTest::Unit::TestCase
+
+
+# ruby stdlibs
+
+require 'json'
+require 'uri'
+require 'pp'
+
+# ruby gems
+
+require 'active_record'
+
+# our own code
+
+require 'worlddb'
+require 'logutils/db'   # NB: explict require required for LogDb (not automatic) 
+
+Country = WorldDb::Models::Country
+Region  = WorldDb::Models::Region
+City    = WorldDb::Models::City
+
+
+def setup_in_memory_db
+  # Database Setup & Config
+
+  db_config = {
+    adapter:  'sqlite3',
+    database: ':memory:'
+  }
+
+  pp db_config
+
+  ActiveRecord::Base.logger = Logger.new( STDOUT )
+  ## ActiveRecord::Base.colorize_logging = false  - no longer exists - check new api/config setting?
+
+  ## NB: every connect will create a new empty in memory db
+  ActiveRecord::Base.establish_connection( db_config )
+
+
+  ## build schema
+
+  LogDb.create
+  WorldDb.create
+end
+
+
+setup_in_memory_db()

data/test/test_values.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_helper.rb
+#  or better
+#     rake test
+
+require 'helper'
+
+class TestValues < MiniTest::Unit::TestCase
+
+  def setup
+    #  delete all countries, regions, cities in in-memory only db
+    WorldDb.delete!
+  end
+
+  def test_load_country_values
+    
+    new_attributes = {
+      key:      'at',
+      title:    'Austria',
+      synonyms: ''
+    }
+
+    values = [
+      'AUT',
+      '83_871',
+      '8_414_638',
+      'un|fifa|uefa|eu|euro|schengen|central_europe|western_europe'
+    ]
+
+    c = Country.create_or_update_from_values( new_attributes, values )
+
+    c2 = Country.find_by_key!( new_attributes[:key] )
+    assert( c.id == c2.id )
+
+    assert( c.title   == new_attributes[:title] )
+    assert( c.pop     == 8_414_638 )
+    assert( c.area    == 83_871 )
+    ## todo: assert tag count; add supra:eu etc.
+  end
+
+  def test_load_region_values
+
+    at = Country.create!( key: 'at',
+                          title: 'Austria',
+                          code: 'AUT',
+                          pop: 8_414_638,
+                          area: 83_871 )
+
+    new_attributes = {
+      key:        'w',
+      title:      'Wien',
+      synonyms:   '',
+      country_id: at.id
+    }
+
+    values = [
+      '415 km²',
+      'eastern austria'
+    ]
+
+    r = Region.create_or_update_from_values( new_attributes, values )
+
+    r2 = Region.find_by_key!( new_attributes[:key] )
+    assert( r.id == r2.id )
+
+    assert( r.title == new_attributes[:title] )
+    assert( r.area  == 415 )
+    ## todo: assert country_id & country.title for assoc
+  end
+
+  def test_load_city_values
+
+    at = Country.create!( key: 'at',
+                          title: 'Austria',
+                          code: 'AUT',
+                          pop: 8_414_638,
+                          area: 83_871 )
+
+    w = Region.create!( key: 'w',
+                        title: 'Wien',
+                        country_id: at.id )
+
+    new_attributes = {
+      key:        'wien',
+      title:      'Wien',
+      synonyms:   '',
+      country_id: at.id
+    }
+
+    values = [
+      'W', 
+      '1_731_236',
+      'm:1_724_000'
+    ]
+
+    c = City.create_or_update_from_values( new_attributes, values )
+
+    c2 = City.find_by_key!( new_attributes[:key] )
+    assert( c.id == c2.id )
+
+    assert( c.title      == new_attributes[:title] )
+    assert( c.pop        == 1_731_236 )
+    assert( c.popm       == 1_724_000 )
+    assert( c.m          == true )
+    assert( c.region_id  == w.id  )
+    assert( c.country_id == at.id )
+  end
+
+
+end # class TestValues
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: worlddb
 version: !ruby/object:Gem::Version
-  version: 1.6.4
+  version: 1.6.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-09 00:00:00.000000000 Z
+date: 2013-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: textutils
-  requirement: &83827050 !ruby/object:Gem::Requirement
+  requirement: &83860400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *83827050
+  version_requirements: *83860400
 - !ruby/object:Gem::Dependency
   name: commander
-  requirement: &83826830 !ruby/object:Gem::Requirement
+  requirement: &83860180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 4.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *83826830
+  version_requirements: *83860180
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &83826620 !ruby/object:Gem::Requirement
+  requirement: &83876350 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '3.2'
   type: :runtime
   prerelease: false
-  version_requirements: *83826620
+  version_requirements: *83876350
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &83826400 !ruby/object:Gem::Requirement
+  requirement: &83876130 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *83826400
+  version_requirements: *83876130
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &83826180 !ruby/object:Gem::Requirement
+  requirement: &83875910 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *83826180
+  version_requirements: *83875910
 description: worlddb - world.db command line tool
 email: opensport@googlegroups.com
 executables:
@@ -99,6 +99,9 @@ files:
 - lib/worlddb/stats.rb
 - lib/worlddb/utils.rb
 - lib/worlddb/version.rb
+- test/helper.rb
+- test/test_values.rb
+- .gemtest
 homepage: https://github.com/geraldb/world.db.ruby
 licenses:
 - Public Domain
@@ -126,4 +129,5 @@ rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: worlddb - world.db command line tool
-test_files: []
+test_files:
+- test/test_values.rb