worlddb 2.0.9 → 2.1.0

data/lib/worlddb/readers/city.rb

@@ -1,81 +0,0 @@
-# encoding: UTF-8
-
-module WorldDb
-
-class CityReader
-
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-
-  def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text, more_attribs )
-  end
-
-
-  def self.from_file( path, more_attribs={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, more_attribs )
-  end
-
-  def self.from_string( text, more_attribs={} )
-    CityReader.new( text, more_attribs )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, more_attribs={} )
-    ## todo/fix: how to add opts={} ???
-
-    @text         = text
-    @more_attribs = more_attribs
-  end
-
-
-  def read()
-    reader = ValuesReader.from_string( @text, @more_attribs )
-
-    reader.each_line do |attribs, values|
-      opts = { skip_tags: skip_tags? }
-      City.create_or_update_from_attribs( attribs, values, opts )
-    end
-  end
-
-end # class CityReader
-end # module WorldDb
-

data/lib/worlddb/readers/country.rb

@@ -1,78 +0,0 @@
-# encoding: UTF-8
-
-module WorldDb
-
-class CountryReader
-
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-
-  def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text, more_attribs )
-  end
-
-  def self.from_file( path, more_attribs={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, more_attribs )    
-  end
-
-  def self.from_string( text, more_attribs={} )
-    CountryReader.new( text, more_attribs )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, more_attribs={} )
-    ## todo/fix: how to add opts={} ???
-
-    @text         = text
-    @more_attribs = more_attribs
-  end
-
-  def read()
-    reader = ValuesReader.from_string( @text, @more_attribs )
-
-    reader.each_line do |attribs, values|
-      opts = { skip_tags: skip_tags? }
-      Country.create_or_update_from_attribs( attribs, values, opts )
-    end
-  end
-
-end # class CountryReader
-end # module WorldDb

data/lib/worlddb/readers/lang.rb

@@ -1,107 +0,0 @@
-# encoding: UTF-8
-
-module WorldDb
-
-class LangReader
-
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-
-  ## todo: add opts={} etc.
-  def self.from_zip( zip_file, entry_path )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text )
-  end
-
-  def self.from_file( path, opts={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, opts )
-  end
-
-  def self.from_string( text, opts={} )
-    LangReader.new( text, opts )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, opts={} )
-    @text = text
-
-    ## option: do NOT generate/add any tags for countries/regions/cities
-    @skip_tags =  opts[:skip_tags].present? ? true : false
-    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
-    @strict    =  opts[:strict].present? ? true : false
-  end
-
-  def read()
-    reader = HashReader.from_string( @text )
-
-    reader.each do |key, value|
-
-      ### fix:
-      ##  move to Lang.read() for (re)use
-
-        logger.debug "adding lang >>#{key}<< >>#{value}<<..."
-      
-        lang_key   = key.strip
-        lang_title = value.strip
-      
-        lang_attribs = {}
-        
-        ## check if it exists
-        lang = Lang.find_by_key( lang_key )
-        if lang.present?
-          logger.debug "update lang #{lang.id}-#{lang.key}:"
-        else
-          logger.debug "create lang:"
-          lang = Lang.new
-          lang_attribs[ :key ] = lang_key
-        end
-        
-        lang_attribs[ :title ] = lang_title
-        
-        logger.debug lang_attribs.to_json
-     
-        lang.update_attributes!( lang_attribs )
-    end # each key,value
-
-  end # method load_langs
-
-
-end # class LangReader
-end # module WorldDb

data/lib/worlddb/readers/region.rb

@@ -1,79 +0,0 @@
-# encoding: UTF-8
-
-module WorldDb
-
-class RegionReader
-
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-
-  def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text, more_attribs )
-  end
-
-  def self.from_file( path, more_attribs={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, more_attribs )
-  end
-
-  def self.from_string( text, more_attribs={} )
-    RegionReader.new( text, more_attribs )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, more_attribs={} )
-    ## todo/fix: how to add opts={} ???
-
-    @text         = text
-    @more_attribs = more_attribs
-  end
-
-
-  def read()
-    reader = ValuesReader.from_string( @text, @more_attribs )
-
-    reader.each_line do |attribs, values|
-      opts = { skip_tags: skip_tags? }
-      Region.create_or_update_from_attribs( attribs, values, opts )
-    end
-  end
-
-end # class RegionReader
-end # module WorldDb

data/lib/worlddb/readers/usage.rb

@@ -1,98 +0,0 @@
-# encoding: UTF-8
-
-module WorldDb
-
-class UsageReader
-
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-  ## todo: add opts
-  def self.from_zip( zip_file, entry_path )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text )
-  end
-
-  def self.from_file( path, opts={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, opts )
-  end
-
-  def self.from_string( text, opts={} )
-    UsageReader.new( text, opts )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, opts={} )
-    @text = text
-
-    ## option: do NOT generate/add any tags for countries/regions/cities
-    @skip_tags =  opts[:skip_tags].present? ? true : false
-    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
-    @strict    =  opts[:strict].present? ? true : false
-  end
-
-  def read()
-    reader = HashReader.from_string( @text )
-
-    reader.each do |key, value|
-      
-      ### fix:
-      ##  move to Usage.read() for (re)use
-
-      logger.debug "   adding langs >>#{value}<<to country >>#{key}<<"
-      
-      country = Country.find_by_key!( key )
-      
-      lang_keys = value.split(',')
-      lang_keys.each do |lang_key|
-
-        ### remove (optional comment) from key (e.g. carribean (islands))
-        lang_key = lang_key.gsub( /\(.+\)/, '' )
-        ## remove leading n trailing space
-        lang_key = lang_key.strip
-
-        lang = Lang.find_by_key!( lang_key )
-        Usage.create!( country_id: country.id, lang_id: lang.id, official: true, minor: false )
-      end
-    end
-  end
-
-
-end # class UsageReader
-end # module WorldDb

data/lib/worlddb/schema.rb

@@ -1,202 +0,0 @@
-
-module WorldDb
-
-class CreateDb 
-
-def up
-
-ActiveRecord::Schema.define do
-
-create_table :places do |t|
-  t.string  :name,   null: false
-  t.string  :kind,   null: false  # --  kind/feature (note: type reserved for activerecord sti)
-  #####
-  # continent:
-  #  CONT - continent (use CNTI or CENT why??)
-  # country:
-  #  SUPR - supra (e.g. European Union)
-  #  CNTY - country
-  #  TERR - terr
-  # region:
-  #  ADM1 - e.g. region/state/province
-  #  ADM2 - e.g. county/district/
-  #  ADM3 - e.g. 
-  # city:
-  #  MTRO - metro
-  #  CITY - city/town/
-  #  DIST - district/
-  #
-  #  add new table for zones (e.g. informal regions e.g. tourism, wine regions, etc.) ??
-  #   why? why not??
-
-
-  t.float   :lat   # optional for now (latitude)
-  t.float   :lng   # optional for now (longitude)
-
-  ## todo: add parent for hierachy ?? or keep it in country/region/city etc. table ??
-
-  ## timestamp at last
-  t.timestamps
-end
-
-
-#############
-# todo: use many-to-many assoc w/ join table  for name and place ???
-#  why? why not?
-#    - wien -> city n region n metro ?  collect more samples of names used more than once
-
-### alternative names for places
-create_table :names do |t|
-  t.string     :name,   null: false
-  t.references :place,  null: false
-  ### todo/fix: add lang_id to reference lang from lang table!!! (for now make it a duplicate; e.g. keep stirng lang for now)
-  t.string     :lang,   null: false, default: 'en'   # default to english for now (or use unknown/undeterminded/unspecified???)
-
-  ## todo: add kind/type   ? e.g. postal for postal code ??
-  ## or wikipedia for wikipedia?  or use separate table ?? (linkable/links) etc.??
-
-  ## timestamp at last
-  t.timestamps
-end
-
-
-
-create_table :continents do |t|
-  t.string     :name,     null: false
-  t.string     :slug,     null: false   # auto-generate default
-  t.string     :key,      null: false
-  t.references :place,    null: false
-  t.string     :alt_names  # comma separated list of alternate names (synonyms)
-
-  ## timestamp at last
-  t.timestamps
-end
-
-add_index :continents, :key, unique: true
-
-
-create_table :country_codes do |t|
-  t.string     :name, null: false
-  t.string     :kind, null: false   # e.g. ALPHA2,NUM,FIFA,IOC,FIPS,NET,etc.
-  t.references :country, null: false
-end
-
-add_index :country_codes, [:name, :kind], unique: true
-
-
-create_table :countries do |t|
-  t.string     :name,   null: false
-  t.string     :slug,   null: false   # auto-generate default
-  t.string     :key,    null: false
-  t.references :place,  null: false
-  t.string     :code,   null: false  # short three letter code (FIFA country code e.g. ITA)
-  t.string     :alt_names  # comma separated list of alternate names (synonyms)
-  t.string     :hist_names # comma separated list of historic names (no longer in use)
-  t.integer    :pop,    null: false    # population count
-  t.integer    :area,   null: false    #  area in square km (sq. km)
-  t.references :continent
-  t.references :country    # for supra(nationals) n depend(encies)
-
-  ## flags (use single int named flags - why? why not?  
-  t.boolean :s,  null: false, default: false   # supra(national) flag e.g. eu
-  t.boolean :c,  null: false, default: false   # country flag (is this needed?)
-  t.boolean :d,  null: false, default: false   # dependency flag
-
-  t.boolean :m,  null: false, default: false   # misc(ellaneous) flag
-  # misc use for territories w/ shared or disputed claims
-  #   e.g. Antartica/Western Sahara/Paracel Islands/Spratly Islands/etc.
-
-  # extras - country codes
-  t.string  :motor      # optional  motor (vehicle) licene plate code (bumper sticker)
-  t.string  :alpha2       # optional iso two letter country code
-  t.string  :alpha3       # optional iso three letter country code
-  t.string  :num        #  optional iso numeric country code - NOTE: same numeric code as string!!! e.g. 043 etc.
-  t.string  :fifa       # optional fifa country code
-  t.string  :ioc
-  t.string  :fips
-  t.string  :net        # optional internet top level domain (tld)
-
-  t.string  :wikipedia  # optional wikipedia page name -- en.wikipedia.org/wiki/<name>
-
-  ## timestamp at last
-  t.timestamps
-end
-
-
-add_index :countries, :key,  unique: true
-add_index :countries, :code, unique: true
-
-
-######
-# NB: rename to adms/admins ??
-#
-# used for state/provice/land/regioni/etc.
-create_table :regions do |t|
-  t.string     :name,   null: false
-  t.string     :key,    null: false
-  t.references :place,  null: false
-  t.string :code     # short two or three letter code e.g. NY, OAX, etc.
-  t.string :abbr     # optional conventional abbrevation (e.g. Stmk., Gto., etc.)
-  t.string :iso      # iso code
-  t.string :nuts     # nuts code (europe/eu only)
-  t.string     :alt_names  # comma separated list of alternate names (synonyms)
-  t.references :country,  null: false
-  t.integer :pop     # optional population count
-  t.integer :area    # optional area in square km (sq. km)
-  t.timestamps
-end
-
-add_index :regions, [:key, :country_id], unique: true
-
-
-create_table :cities do |t|
-  t.string     :name,   null: false
-  t.string     :key,    null: false
-  t.references :place,  null: false
-  t.string     :code     # short three letter code (ITAT/airport code e.g. NYC or VIE)
-  t.string     :alt_names  # comma separated list of alternate names (synonyms)
-  t.references :country,  null: false
-  t.references :region    # optional for now
-  t.references :city  # optional parent (e.g. metro for city, or city for district)
-  t.integer :pop     # optional population count (city proper)
-  t.integer :popm    # optional population count (metropolitan/aglomeration)
-  t.integer :area    # optional area in square km (sq. km)
-
-  ## t.float   :lat   # optional for now  --  FIX: remove?? moved to places
-  ## t.float   :lng   # optional for now  --  FIX: remove?? moved to places
-
-  ## flags (use single int named flags - why? why not?    
-  t.boolean :m,  null: false, default: false   # metro flag
-  t.boolean :c,  null: false, default: false   # city flag (is this needed?)
-  t.boolean :d,  null: false, default: false   # district flag
-
-  ### t.boolean :capital, null: false, default: false  # is national captial?
-
-  t.timestamps
-end
-
-
-create_table :langs do |t|  # langs == languages (e.g. en/English, de/Deutsch, etc.)
-  t.string :key,   null: false
-  t.string :name,  null: false
-  t.timestamps
-end
-
-
-create_table :usages do |t|  # join table for countries_langs
-  t.references :country,  null: false
-  t.references :lang,     null: false
-  t.boolean    :official, null: false, default: true  # is_official language in country
-  t.boolean    :minor,    null: false, default: false # spoken by minority 
-  t.float      :percent     # usage in percent e.g. 90.0, 0.55, etc.
-  t.timestamps
-end
-
-end # block Schema.define
-
-end # method up
-
-
-end # class CreateDb
-
-end # module WorldDb