worlddb 0.8.0 → 1.0.0

data/lib/worlddb/reader.rb

@@ -9,8 +9,7 @@ class Reader
 
   def initialize( logger=nil )
     if logger.nil?
-      @logger = Logger.new(STDOUT)
-      @logger.level = Logger::INFO
+      @logger = LogUtils::Logger.new
     else
       @logger = logger
     end
@@ -18,26 +17,6 @@ class Reader
 
   attr_reader :logger
 
-  def run( opts, args )
- 
-    args.each do |arg|
-      name = arg     # File.basename( arg, '.*' )
-
-      data_path = opts.load? ? WorldDB.data_path : opts.data_path
-
-      if opts.countries?
-        load_countries_with_include_path( name, data_path )
-      elsif opts.regions?
-        load_regions_with_include_path( opts.country, name, data_path )
-      elsif opts.cities?
-        load_cities_with_include_path( opts.country, name, data_path )
-      else
-        ## todo: issue a warning here; no fixture type specified; assume country?
-      end
-    end # each arg
-
-  end
-
 
   def load_with_include_path( name, include_path )
 
@@ -65,60 +44,43 @@ class Reader
       ## auto-add required country code (from folder structure)
       load_regions_with_include_path( $1, name, include_path )
     else
-      puts "*** error: unknown world.db fixture type >#{name}<"
+      logger.error "unknown world.db fixture type >#{name}<"
       # todo/fix: exit w/ error
     end
   end
   
 
-  def load_builtin( name )  ## convenience helper (requires proper named files w/ convention)
-    load_with_include_path( name, WorldDB.data_path )
-  end
-
-
   def load_countries_with_include_path( name, include_path, more_values={} )
     load_fixtures_with_include_path_for( Country, name, include_path, more_values )
   end
 
-  def load_countries_builtin( name, more_values={} )
-    load_countries_with_include_path( name, WorldDB.data_path, more_values )
-  end
-
 
   def load_regions_with_include_path( country_key, name, include_path )
     country = Country.find_by_key!( country_key )
-    puts "Country #{country.key} >#{country.title} (#{country.code})<"
+    logger.info "Country #{country.key} >#{country.title} (#{country.code})<"
 
     load_fixtures_with_include_path_for( Region, name, include_path, country_id: country.id )
   end
 
-  def load_regions_builtin( country_key, name )
-    load_regions_with_include_path( country_key, name, WorldDB.data_path )
-  end
-
 
   def load_cities_with_include_path( country_key, name, include_path )
     country = Country.find_by_key!( country_key )
-    puts "Country #{country.key} >#{country.title} (#{country.code})<"
+    logger.info "Country #{country.key} >#{country.title} (#{country.code})<"
 
     load_fixtures_with_include_path_for( City, name, include_path, country_id: country.id )
   end
 
-  def load_cities_builtin( country_key, name )
-    load_cities_with_include_path( country_key, name, WorldDB.data_path )
-  end
-
 
   def load_langs_with_include_path( name, include_path )
     path = "#{include_path}/#{name}.yml"
 
-    puts "*** parsing data '#{name}' (#{path})..."
+    logger.info "*** parsing data '#{name}' (#{path})..."
 
     reader = HashReader.new( logger, path )
 
     reader.each do |key, value|
 
-      puts "adding lang >>#{key}<< >>#{value}<<..."
+      logger.debug "adding lang >>#{key}<< >>#{value}<<..."
       
       lang_key   = key.strip
       lang_title = value.strip
@@ -144,10 +106,6 @@ class Reader
 
     Prop.create_from_worlddb_fixture!( name, path )
   end
-  
-  def load_langs_builtin( name )
-    load_langs_with_include_path( name, WorldDB.data_path )
-  end
 
 
   def load_tags_with_include_path( name, include_path, more_values={} )
@@ -205,10 +163,6 @@ class Reader
     Prop.create_from_worlddb_fixture!( name, path )
   end # method load_tags_with_include_path
 
-  def load_tags_builtin( name, include_path, more_values={} )
-    load_tags_with_include_path( name, WorldDB.data_path, more_values )
-  end
-
 
   def load_usages_with_include_path( name, include_path )
     path = "#{include_path}/#{name}.yml"
@@ -237,10 +191,7 @@ class Reader
 
     Prop.create_from_worlddb_fixture!( name, path )
   end
-  
-  def load_usages_builtin( name )
-    load_usages_with_include_path( name, WorldDB.data_path )
-  end
+
 
   def load_xxx_with_include_path( xxx, name, include_path )
     path = "#{include_path}/#{name}.yml"
@@ -258,10 +209,6 @@ class Reader
     Prop.create_from_worlddb_fixture!( name, path )
   end
 
-  def load_xxx_builtin( xxx, name )
-    load_xxx_with_include_path( xxx, name, WorldDB.data_path )
-  end
-
 
 private
   def load_fixtures_with_include_path_for( clazz, name, include_path, more_values={} )  # load from file system
@@ -380,7 +327,7 @@ private
           end
 
           # issue warning: unknown type for value
-          puts "!!!! >>>> warning: unknown type for value >#{value}<"
+          logger.warn "unknown type for value >#{value}<"
         end
       end # each value
 

data/lib/worlddb/stats.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+
+module WorldDB
+
+  class Stats
+    include WorldDB::Models
+
+    def tables
+      puts "Stats:"
+      puts " #{'%5d' % Country.count} countries (#{Country.where(s: true).count} supras, #{Country.where(d:true).count} deps)"
+      puts " #{'%5d' % Region.count} regions"
+      puts " #{'%5d' % City.where(m: true).where(c: false).count} metros"
+      puts " #{'%5d' % City.where(c: true).count} cities (#{City.where(c: true).where(m: true).count} metros)"
+      puts " #{'%5d' % City.where(d: true).count} districts"
+      puts " #{'%5d' % Tag.count} tags"
+      puts " #{'%5d' % Tagging.count} taggings"
+      puts " #{'%5d' % Lang.count} langs"
+      puts " #{'%5d' % Usage.count} usages"
+    end
+    
+    def props
+      puts "Props:"
+      Prop.order( 'created_at asc' ).all.each do |prop|
+        puts "  #{prop.key} / #{prop.value} || #{prop.created_at}"
+      end
+    end
+  end # class Stats
+
+
+end # module WorldDB

data/lib/worlddb/utils.rb

@@ -16,63 +16,3 @@ class Time
   end
   
 end # class Time
-
-
-class File
-  def self.read_utf8( path )
-    open( path, 'r:bom|utf-8' ) do |file|
-      file.read
-    end
-  end
-end # class File
-
-  
-  ############
-  ### fix/todo: share helper for all text readers/parsers- where to put it?  
-  ###
-  
-  def title_esc_regex( title_unescaped )
-      
-      ##  escape regex special chars e.g. . to \. and ( to \( etc.
-      # e.g. Benfica Lis.
-      # e.g. Club Atlético Colón (Santa Fe)
-
-      ## NB: cannot use Regexp.escape! will escape space '' to '\ '
-      ## title = Regexp.escape( title_unescaped )
-      title = title_unescaped.gsub( '.', '\.' )
-      title = title.gsub( '(', '\(' )
-      title = title.gsub( ')', '\)' )
-
-      ##  match accented char with or without accents
-      ##  add (ü|ue) etc.
-      ## also make - optional change to (-| ) e.g. Blau-Weiss == Blau Weiss
-
-      ## todo: add some more
-      ## see http://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references  for more
-      ##
-      ##  reuse for all readers!
-      
-      alternatives = [
-        ['-', '(-| )'],
-        ['ß', '(ß|ss)'],
-        ['æ', '(æ|ae)'],
-        ['á', '(á|a)'],  ## e.g. Bogotá
-        ['ã', '(ã|a)'],  ## e.g  São Paulo
-        ['ä', '(ä|ae)'],  ## add a ?
-        ['Ö', '(Ö|Oe)'], ## e.g. Österreich
-        ['ö', '(ö|oe)'],  ## add o ?
-        ['ó', '(ó|o)'],  ## e.g. Colón
-        ['ü', '(ü|ue)'],  ## add u ?
-        ['é', '(é|e)'],  ## e.g. Vélez
-        ['ê', '(ê|e)'],  ## e.g. Grêmio
-        ['ñ', '(ñ|n)'],  ## e.g. Porteño
-        ['ú', '(ú|u)']  ## e.g. Fútbol
-      ]
-      
-      alternatives.each do |alt|
-        title = title.gsub( alt[0], alt[1] )
-      end
-
-      title
-  end
-

data/lib/worlddb/version.rb

@@ -1,5 +1,4 @@
 
 module WorldDB
-  VERSION = '0.8.0'
+  VERSION = '1.0.0'  # sync version w/ sport.db - why? why not?
 end
-

data/lib/worlddb.rb

@@ -12,13 +12,15 @@ require 'optparse'
 require 'fileutils'
 require 'erb'
 
-# rubygems
+# 3rd party gems / libs
 
 require 'active_record'   ## todo: add sqlite3? etc.
 
+require 'logutils'
+require 'textutils'
 
-# our own code
 
+# our own code
 
 require 'worlddb/version'
 
@@ -32,13 +34,13 @@ require 'worlddb/models/lang'
 require 'worlddb/models/usage'
 require 'worlddb/schema'       # NB: requires worlddb/models (include WorldDB::Models)
 require 'worlddb/utils'
-require 'worlddb/readers/code_reader'
-require 'worlddb/readers/line_reader'
-require 'worlddb/readers/values_reader'
-require 'worlddb/readers/hash_reader'
 require 'worlddb/reader'
-require 'worlddb/cli/opts'
-require 'worlddb/cli/runner'
+require 'worlddb/deleter'
+require 'worlddb/stats'
+
+require 'worlddb/data/fixtures'
+
+
 
 module WorldDB
 
@@ -49,147 +51,25 @@ module WorldDB
   def self.root
     "#{File.expand_path( File.dirname(File.dirname(__FILE__)) )}"
   end
-  
-  # builtin path to fixture data
-  def self.data_path
-    "#{root}/data"
-  end
 
   def self.main
-    Runner.new.run(ARGV)
+    require 'worlddb/cli/main'
+    ## Runner.new.run(ARGV) - old code
   end
   
   def self.create
     CreateDB.up
   end
 
-  def self.fixtures  # all builtin fixtures; helper for covenience
-     tag_fixtures  + 
-     country_fixtures +
-     lang_fixtures +
-     africa_fixtures  +
-     america_fixtures +
-     europe_fixtures  +
-     asia_fixtures    +
-     oceania_fixtures
-  end
-
-
-  def self.tag_fixtures
-    ['tags.1', 'tags.3']
-  end
-  
-  def self.country_fixtures
-    ['africa/countries',
-     'america/countries',
-     'asia/countries',
-     'europe/countries',
-     'oceania/countries']
-  end
-
-  def self.lang_fixtures
-    ['langs',
-     'africa/3_more/lang',
-     'america/3_more/lang',
-     'asia/3_more/lang',
-     'europe/3_more/lang',
-     'oceania/3_more/lang'
-    ]
-  end
-
-  def self.africa_fixtures
-    ['1_codes/fifa',
-     '1_codes/internet',
-     '1_codes/iso3'].map { |path| "africa/#{path}" }
-  end
-
-  def self.america_fixtures
-  ['1_codes/fifa',
-   '1_codes/internet',
-   '1_codes/iso3',
-   '1_codes/motor',
-   'br/regions',
-   'ca/regions',
-   'ca/cities',
-   'mx/regions',
-   'mx/cities',
-   'us/regions',
-   'us/cities',
-   've/regions',
-   've/cities'].map { |path| "america/#{path}" }
-  end
-
-  def self.asia_fixtures
-  ['1_codes/fifa',
-   '1_codes/internet',
-   '1_codes/iso3',
-   'jp/cities'].map { |path| "asia/#{path}" }
-  end
-  
-  def self.europe_fixtures
-  ['1_codes/fifa',
-   '1_codes/internet',
-   '1_codes/iso3',
-   '1_codes/motor',
-   'at/regions',
-   'at/cities',
-   'be/regions',
-   'be/cities',
-   'bg/cities',
-   'by/cities',
-   'ch/cities',
-   'cy/cities',
-   'cz/regions',
-   'cz/cities',
-   'de/regions',
-   'de/cities',
-   'dk/cities',
-   'ee/cities',
-   'en/regions',
-   'en/cities',
-   'es/regions',
-   'es/cities',
-   'fi/cities',
-   'fr/regions',
-   'fr/cities',
-   'gr/cities',
-   'hr/cities',
-   'hu/cities',
-   'ie/cities',
-   'it/cities',
-   'lt/cities',
-   'lv/cities',
-   'nl/cities',
-   'no/cities',
-   'pl/cities',
-   'pt/cities',
-   'ro/cities',
-   'rs/cities',
-   'ru/cities',
-   'sc/cities',
-   'se/cities',
-   'tr/cities',
-   'ua/cities',
-   'wa/cities'].map { |path| "europe/#{path}" }
-  end
-
-  def self.oceania_fixtures
-   ['1_codes/fifa',
-    '1_codes/internet',
-    '1_codes/iso3',
-    'au/cities'].map { |path| "oceania/#{path}" }
-  end
-
-  ## todo/fix: rename to load/load_all - why? why not?? or just add an alias?
 
-  def self.read( ary )
+  def self.read( ary, include_path )
     reader = Reader.new
     ary.each do |name|
-      reader.load_builtin( name )
+      reader.load_with_include_path( name, include_path )
     end
   end
 
-  def self.read_all  # load all builtins (using plain text reader); helper for convenience
+  def self.read_all( include_path )  # load all builtins (using plain text reader); helper for convenience
     reader = Reader.new
 
     # too big for heroku free db plan (10,000 record limit)
@@ -198,35 +78,18 @@ module WorldDB
       'america/ve/cities'
     ]
     
-    ary = fixtures - fixture_excludes
+    ary = Fixtures.all - fixture_excludes
     
     ary.each do |name|
-     reader.load_builtin( name )
+     reader.load_with_include_path( name, include_path )
     end # each name
-  end # method load_all
-
-
-  class Deleter
-    ## todo: move into its own file???    
-    
-    ## make models available in worlddb module by default with namespace
-    #  e.g. lets you use City instead of Models::City 
-    include WorldDB::Models
-
-    def run( args=[] )
-      # for now delete all tables
-      
-      Tagging.delete_all
-      Tag.delete_all
-      City.delete_all
-      Region.delete_all
-      Country.delete_all
-      Usage.delete_all
-      Lang.delete_all
-      Prop.delete_all
-    end
-    
+  end # method read_all
+  
+  
+  def self.read_setup( setup, include_path )
+    ## fix: to be done  (see sport.db.ruby)
   end
+
   
   # delete ALL records (use with care!)
   def self.delete!
@@ -234,30 +97,7 @@ module WorldDB
     Deleter.new.run
   end # method delete!
 
-  class Stats
-    include WorldDB::Models
-
-    def tables
-      puts "Stats:"
-      puts " #{'%5d' % Country.count} countries (#{Country.where(s: true).count} supras, #{Country.where(d:true).count} deps)"
-      puts " #{'%5d' % Region.count} regions"
-      puts " #{'%5d' % City.where(m: true).where(c: false).count} metros"
-      puts " #{'%5d' % City.where(c: true).count} cities (#{City.where(c: true).where(m: true).count} metros)"
-      puts " #{'%5d' % City.where(d: true).count} districts"
-      puts " #{'%5d' % Tag.count} tags"
-      puts " #{'%5d' % Tagging.count} taggings"
-      puts " #{'%5d' % Lang.count} langs"
-      puts " #{'%5d' % Usage.count} usages"
-    end
-    
-    def props
-      puts "Props:"
-      Prop.order( 'created_at asc' ).all.each do |prop|
-        puts "  #{prop.key} / #{prop.value} || #{prop.created_at}"
-      end
-    end
-  end
-
+ 
   def self.stats
     stats = Stats.new
     stats.tables