worlddb-models 2.1.0

data/lib/worlddb/models/region.rb

@@ -0,0 +1,176 @@
+# encoding: UTF-8
+
+module WorldDb
+  module Model
+
+
+class Region < ActiveRecord::Base
+
+  extend TextUtils::TagHelper  # will add self.find_tags, self.find_tags_in_attribs!, etc.
+
+  # NB: use extend - is_<type>? become class methods e.g. self.is_<type>? for use in
+  #   self.create_or_update_from_values
+  extend TextUtils::ValueHelper  # e.g. is_year?, is_region?, is_address?, is_taglist? etc.
+
+  belongs_to :place,   class_name: 'Place',   foreign_key: 'place_id'
+  belongs_to :country, class_name: 'Country', foreign_key: 'country_id'
+
+  has_many :cities, class_name: 'City', foreign_key: 'region_id'
+
+  has_many_tags
+
+  validates :key,  format: { with: /#{REGION_KEY_PATTERN}/,  message: REGION_KEY_PATTERN_MESSAGE }
+  validates :code, format: { with: /#{REGION_CODE_PATTERN}/, message: REGION_CODE_PATTERN_MESSAGE }, allow_nil: true
+
+
+  before_create :on_create
+  before_update :on_update
+
+  def on_create
+    place_rec = Place.create!( name: name, kind: place_kind )
+    self.place_id = place_rec.id 
+  end
+
+  def on_update
+    ## fix/todo: check - if name or kind changed - only update if changed ?? why? why not??
+    place.update_attributes!( name: name, kind: place_kind )
+  end
+
+  def place_kind   # use place_kind_of_code ??
+    'ADM1'
+  end
+
+
+  def all_names( opts={} )
+    ### fix:
+    ## allow to passing in sep or separator e.g. | or other
+
+    return name if alt_names.blank?
+    
+    buf = ''
+    buf << name
+    buf << ' | '
+    buf << alt_names.split('|').join(' | ')
+    buf
+  end
+
+
+
+  def self.create_or_update_from_values( values, more_attribs={} )
+
+    ## key & title & country required
+    attribs, more_values = find_key_n_title( values )
+    attribs = attribs.merge( more_attribs )
+
+    ## check for optional values
+    Region.create_or_update_from_attribs( attribs, more_values )
+  end
+
+
+  def self.create_or_update_from_attribs( new_attributes, values, opts={} )
+
+    ## opts e.g. :skip_tags true|false
+
+    ## fix: add/configure logger for ActiveRecord!!!
+    logger = LogKernel::Logger.root
+
+    value_numbers     = []
+    value_tag_keys    = []
+    value_cities      = []
+
+    ### check for "default" tags - that is, if present new_attributes[:tags] remove from hash
+    value_tag_keys += find_tags_in_attribs!( new_attributes )
+
+    ## check for optional values
+    values.each_with_index do |value,index|
+      if match_country( value ) do |country| # country:
+           new_attributes[ :country_id ] = country.id
+         end
+      elsif match_km_squared( value ) do |num| # allow numbers like 453 km²
+              value_numbers << num
+            end
+      elsif match_number( value ) do |num|  # numeric (nb: can use any _ or spaces inside digits e.g. 1_000_000 or 1 000 000)
+              value_numbers << num
+            end
+      elsif value =~ /#{REGION_CODE_PATTERN}/  ## assume two or three-letter code
+        new_attributes[ :code ] = value
+      elsif (values.size==(index+1)) && is_taglist?( value )   # tags must be last entry
+        logger.debug "   found tags: >>#{value}<<"
+        value_tag_keys += find_tags( value )
+      else
+          
+        ### assume it is the capital city - mark it for auto add
+        value_cities << value
+        next
+   
+        # issue warning: unknown type for value
+        # logger.warn "unknown type for value >#{value}<"
+      end
+    end # each value
+
+    if value_numbers.size > 0
+      new_attributes[ :area ] = value_numbers[0]
+      new_attributes[ :pop  ] = value_numbers[1]
+    end  # if value_numbers.size > 0
+
+    ## todo: assert that country_id is present/valid, that is, NOT null
+    rec = Region.find_by_key_and_country_id( new_attributes[ :key ], new_attributes[ :country_id] )
+
+    if rec.present?
+      logger.debug "update Region #{rec.id}-#{rec.key}:"
+    else
+      logger.debug "create Region:"
+      rec = Region.new
+    end
+      
+    logger.debug new_attributes.to_json
+   
+    rec.update_attributes!( new_attributes )
+
+    #################
+    # auto add capital cities
+
+    City.create_or_update_from_titles( value_cities,
+                                            region_id:  rec.id,
+                                            country_id: rec.country_id )
+
+    ### todo/fix: add captial ref to country/region
+    ## todo/fix: use update_from_title and only allow one capital city
+
+
+    ##################
+    # add taggings 
+
+    ## todo/fix: reuse - move add taggings into method etc.
+
+      if value_tag_keys.size > 0
+        
+        if opts[:skip_tags].present?
+          logger.debug "   skipping add taggings (flag skip_tag)"
+        else
+          value_tag_keys.uniq!  # remove duplicates
+          logger.debug "   adding #{value_tag_keys.size} taggings: >>#{value_tag_keys.join('|')}<<..."
+
+          ### fix/todo: check tag_ids and only update diff (add/remove ids)
+
+          value_tag_keys.each do |key|
+            tag = Tag.find_by_key( key )
+            if tag.nil?  # create tag if it doesn't exit
+              logger.debug "   creating tag >#{key}<"
+              tag = Tag.create!( key: key )
+            end
+            rec.tags << tag
+          end
+        end
+      end
+
+    rec
+
+  end # method create_or_update_from_values
+
+
+
+end # class Region
+
+  end # module Model
+end # module WorldDb

data/lib/worlddb/models/region_comp.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+
+module WorldDb
+  module Model
+
+#############################################################
+# collect depreciated or methods for future removal here
+#  - keep for now for commpatibility (for old code)
+
+class Region
+  
+  def title()       name;              end
+  def title=(value) self.name = value; end
+
+  scope :by_title, ->{ order( 'name asc' ) } # order by title (a-z)
+
+  def synonyms()       alt_names;      end
+  def synonyms=(value) self.alt_names = value; end
+
+  def title_w_synonyms( opts={} )  all_names( opts );  end    # depreciated: use all_names instead
+
+
+end # class Region
+
+  end # module Model
+end # module WorldDb

data/lib/worlddb/models/tagdb/tag.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+
+module TagDb
+  module Model
+
+class Tag
+
+  has_many :cities,    :through => :taggings, :source => :taggable, source_type: 'WorldDb::Model::City',    class_name: 'WorldDb::Model::City'
+  has_many :countries, :through => :taggings, :source => :taggable, source_type: 'WorldDb::Model::Country', class_name: 'WorldDb::Model::Country'
+  has_many :regions,   :through => :taggings, :source => :taggable, source_type: 'WorldDb::Model::Region',  class_name: 'WorldDb::Model::Region'
+
+end  # class Tag
+
+  end # module Model
+end # module WorldDb
+

data/lib/worlddb/models/tagdb/tagging.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+
+module TagDb
+  module Model
+
+
+class Tagging
+
+  ## add some code here
+
+end  # class Tagging
+
+
+  end # module Model
+end # module WorldDb

data/lib/worlddb/models/usage.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+
+module WorldDb
+  module Model
+
+
+class Usage < ActiveRecord::Base
+
+  belongs_to :country
+  belongs_to :lang
+
+end  # class Usage
+
+
+  end # module Model
+end # module WorldDb
+

data/lib/worlddb/models.rb

@@ -0,0 +1,200 @@
+# encoding: UTF-8
+
+# core and stlibs
+
+require 'pp'
+require 'fileutils'
+require 'uri'
+require 'erb'
+require 'json'
+require 'yaml'
+
+
+# 3rd party gems / libs
+
+require 'zip'       ## rubyzip gem
+
+require 'props'           # see github.com/rubylibs/props
+require 'logutils'        # see github.com/rubylibs/logutils
+require 'textutils'       # see github.com/rubylibs/textutils
+
+
+require 'active_record'   ## todo: add sqlite3? etc.
+
+## add more activerecords addons/utils
+require 'tagutils'
+require 'activerecord/utils'
+require 'props/activerecord'      # includes ConfDb (ConfDb::Model::Prop, etc.)
+require 'logutils/activerecord'   # includes LogDb (LogDb::Model::Log, etc.)
+
+
+
+# our own code
+
+require 'worlddb/version'   # always goes first
+
+require 'worlddb/patterns'  # regex patterns (constants)
+require 'worlddb/models/forward'
+
+require 'worlddb/models/name'
+require 'worlddb/models/place'
+require 'worlddb/models/continent'
+require 'worlddb/models/continent_comp'
+require 'worlddb/models/country'
+require 'worlddb/models/country_comp'
+require 'worlddb/models/country_code'
+require 'worlddb/models/region'
+require 'worlddb/models/region_comp'
+require 'worlddb/models/city'
+require 'worlddb/models/city_comp'
+require 'worlddb/models/lang'
+require 'worlddb/models/lang_comp'
+require 'worlddb/models/usage'
+
+require 'worlddb/models/tagdb/tag'
+require 'worlddb/models/tagdb/tagging'
+
+
+
+require 'worlddb/schema'       # NB: requires worlddb/models (include WorldDB::Models)
+require 'worlddb/matcher'
+
+require 'worlddb/readers/lang'
+require 'worlddb/readers/usage'
+require 'worlddb/readers/country'
+require 'worlddb/readers/region'
+require 'worlddb/readers/city'
+
+require 'worlddb/reader'
+require 'worlddb/reader_file'
+require 'worlddb/reader_zip'
+require 'worlddb/deleter'
+require 'worlddb/stats'
+
+
+module WorldDb
+
+  def self.create
+    CreateDb.new.up
+    ConfDb::Model::Prop.create!( key: 'db.schema.world.version', value: VERSION )
+  end
+
+  def self.create_all
+    LogDb.create    # add logs table
+    ConfDb.create   # add props table
+    TagDb.create    # add tags, taggings table
+    WorldDb.create
+  end
+
+
+  def self.read( ary, include_path )
+    reader = Reader.new( include_path )
+    ary.each do |name|
+      reader.load( name )
+    end
+  end
+
+
+  def self.read_setup( setup, include_path, opts={} )
+    reader = Reader.new( include_path, opts )
+    reader.load_setup( setup )
+  end
+
+  def self.read_setup_from_zip( zip_name, setup, include_path, opts={} ) ## todo/check - use a better (shorter) name ??
+    reader = ZipReader.new( zip_name, include_path, opts )
+    reader.load_setup( setup )
+    reader.close
+  end
+
+  def self.read_all( include_path, opts={} )  # load all builtins (using plain text reader); helper for convenience
+    read_setup( 'setups/all', include_path, opts )
+  end # method read_all
+
+
+  # delete ALL records (use with care!)
+  def self.delete!
+    puts '*** deleting world table records/data...'
+    Deleter.new.run
+  end # method delete!
+
+  def self.delete_all!( opts={} )
+    LogDb.delete!
+    ConfDb.delete!
+    TagDb.delete!
+    WorldDb.delete!
+  end
+
+
+ ####
+ ## todo: remove stats ??? why? why not? better use .tables
+  def self.stats
+    Stats.new.tables
+  end
+
+  def self.tables
+    Stats.new.tables
+  end
+
+
+  def self.connect( db_config={} )
+
+    if db_config.empty?
+      puts "ENV['DATBASE_URL'] - >#{ENV['DATABASE_URL']}<"
+
+      ### change default to ./sport.db ?? why? why not?
+      db = URI.parse( ENV['DATABASE_URL'] || 'sqlite3:///world.db' )
+
+      if db.scheme == 'postgres'
+        config = {
+          adapter: 'postgresql',
+          host: db.host,
+          port: db.port,
+          username: db.user,
+          password: db.password,
+          database: db.path[1..-1],
+          encoding: 'utf8'
+        }
+      else # assume sqlite3
+       config = {
+         adapter: db.scheme, # sqlite3
+         database: db.path[1..-1] # world.db (NB: cut off leading /, thus 1..-1)
+      }
+      end
+    else
+      config = db_config  # use passed in config hash
+    end
+
+    ## todo/check: use if defined?( JRUBY_VERSION ) instead ??
+    if RUBY_PLATFORM =~ /java/ && config[:adapter] == 'sqlite3' 
+      # quick hack for JRuby sqlite3 support via jdbc
+      puts "jruby quick hack - adding jdbc libs for jruby sqlite3 database support"
+      require 'jdbc/sqlite3'
+      require 'active_record/connection_adapters/jdbc_adapter'
+      require 'active_record/connection_adapters/jdbcsqlite3_adapter'
+    end
+
+    puts "Connecting to db using settings: "
+    pp config
+    ActiveRecord::Base.establish_connection( config )
+    # ActiveRecord::Base.logger = Logger.new( STDOUT )
+  end
+
+
+  def self.setup_in_memory_db
+
+    # Database Setup & Config
+    ActiveRecord::Base.logger = Logger.new( STDOUT )
+    ## ActiveRecord::Base.colorize_logging = false  - no longer exists - check new api/config setting?
+
+    self.connect( adapter:  'sqlite3',
+                  database: ':memory:' )
+
+    ## build schema
+    WorldDb.create_all
+  end # setup_in_memory_db (using SQLite :memory:)
+
+end  # module WorldDb
+
+
+# say hello
+puts WorldDb.banner    if $DEBUG || (defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG)

data/lib/worlddb/patterns.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+
+module WorldDb
+
+# collection of regex patterns for reuse (WorldDb specific)
+
+### todo: add a patterns.md page to  github ??
+##  - add regexper pics??
+
+############
+# about ruby regexps
+#
+# try the rubular - Ruby regular expression editor and tester
+#  -> http://rubular.com
+#   code -> ??  by ??
+#
+#
+# Jeff Avallone's Regexper - Shows State-Automata Diagrams
+#  try -> http://regexper.com
+#    code -> https://github.com/javallone/regexper
+#
+#
+#  Regular Expressions | The Bastards Book of Ruby by Dan Nguyen
+# http://ruby.bastardsbook.com/chapters/regexes/
+#
+# move to notes  regex|patterns on  geraldb.github.io ??
+#
+
+  COUNTRY_KEY_PATTERN = '\A[a-z]{2,3}\z'    # allow two AND three letter keys e.g. at, mx, eng, sco, etc.
+  COUNTRY_KEY_PATTERN_MESSAGE = "expected two or three lowercase letters a-z /#{COUNTRY_KEY_PATTERN}/"
+
+  COUNTRY_CODE_PATTERN = '\A[A-Z_]{3}\z'
+  COUNTRY_CODE_PATTERN_MESSAGE = "expected three uppercase letters A-Z (and _) /#{COUNTRY_CODE_PATTERN}/"
+
+
+  REGION_KEY_PATTERN = '\A[a-z]+\z'
+  REGION_KEY_PATTERN_MESSAGE = "expected one or more lowercase letters a-z /#{REGION_KEY_PATTERN}/"
+
+  REGION_CODE_PATTERN = '\A[A-Z_]{2,3}\z'
+  REGION_CODE_PATTERN_MESSAGE = "expected two or three uppercase letters A-Z (and _) /#{REGION_CODE_PATTERN}/"
+
+
+  CITY_KEY_PATTERN = '\A[a-z]{3,}\z'
+  CITY_KEY_PATTERN_MESSAGE = "expected three or more lowercase letters a-z' /#{CITY_KEY_PATTERN}/"
+  
+  CITY_CODE_PATTERN = '\A[A-Z_]{3}\z'
+  CITY_CODE_PATTERN_MESSAGE = "expected three uppercase letters A-Z (and _)' /#{CITY_CODE_PATTERN}/"
+
+
+  LANG_KEY_PATTERN = '\A[a-z]{2}\z'
+  LANG_KEY_PATTERN_MESSAGE = "expected two lowercase letters a-z' /#{LANG_KEY_PATTERN}/"
+
+end # module WorldDb
+