worlddb 0.6.8 → 0.7.0

data/Manifest.txt

@@ -102,6 +102,7 @@ lib/worlddb/models/region.rb
 lib/worlddb/models/tag.rb
 lib/worlddb/models/tagging.rb
 lib/worlddb/reader.rb
+lib/worlddb/readers/code_reader.rb
 lib/worlddb/readers/hash_reader.rb
 lib/worlddb/readers/line_reader.rb
 lib/worlddb/readers/values_reader.rb

data/Rakefile

@@ -1,9 +1,17 @@
+require 'pp'
 require 'hoe'
 require './lib/worlddb/version.rb'
 
 ## NB: plugin (hoe-manifest) not required; just used for future testing/development
 Hoe::plugin :manifest   # more options for manifests (in the future; not yet)
 
+
+###########
+#### NB: if you try this script at home
+#    you need to create a (symbolic) link to the world.db fixtures
+#     e.g. use ln -s ../world.db  data  or similar
+
+
 Hoe.spec 'worlddb' do
   
   self.version = WorldDB::VERSION
@@ -15,11 +23,6 @@ Hoe.spec 'worlddb' do
   
   self.author  = 'Gerald Bauer'
   self.email   = 'opensport@googlegroups.com'
-    
-  # switch extension to .markdown for gihub formatting
-  #  - auto-added from Manifest.txt (see define_spec in class Hoe)
-  # self.readme_file  = 'README.md'
-  # self.history_file = 'History.md'
   
   self.extra_deps = [
     ['activerecord', '~> 3.2']  # NB: will include activesupport,etc.
@@ -32,4 +35,49 @@ Hoe.spec 'worlddb' do
     :required_ruby_version => '>= 1.9.2'
   }
 
-end
+end
+
+##############################
+## for testing 
+##
+## NB: use rake -I ./lib dev:test      # fresh import (starts w/ clean wipe out)
+
+namespace :dev do
+  
+  BUILD_DIR = "./build"
+  
+  WORLD_DB_PATH = "#{BUILD_DIR}/world.db"
+
+  DB_CONFIG = {
+    :adapter   =>  'sqlite3',
+    :database  =>  WORLD_DB_PATH
+  }
+
+  directory BUILD_DIR
+
+  task :clean do
+    rm WORLD_DB_PATH if File.exists?( WORLD_DB_PATH )
+  end
+
+  task :env => BUILD_DIR do
+    require './lib/worlddb.rb'
+
+    pp DB_CONFIG
+    ActiveRecord::Base.establish_connection( DB_CONFIG )
+  end
+
+  task :create => :env do
+    WorldDB.create
+  end
+  
+  task :import => :env do
+    WorldDB.read_all  # populate world tables
+    WorldDB.stats
+  end
+
+
+  desc 'worlddb - test loading of builtin fixtures'
+  task :test => [:clean, :create, :import] 
+
+
+end  # namespace :dev

data/lib/worlddb/loader.rb

@@ -2,11 +2,6 @@ module WorldDB
 
 class Loader
 
-## make models available in worlddb module by default with namespace
-#  e.g. lets you use City instead of Models::City 
-  include WorldDB::Models
-
-
   def initialize( logger=nil )
     if logger.nil?
       @logger = Logger.new(STDOUT)
@@ -34,42 +29,27 @@ class Loader
   end # method run
 
 
+  class CodeContext
+    ## make models available in worlddb module by default with namespace
+    #  e.g. lets you use City instead of Models::City 
+    include WorldDB::Models
+  end
+
+  ### todo: rename to load_fixtures_w_include_path (a little shorter - why? why not?)
   def load_fixtures_with_include_path( name, include_path )  # load from file system
     path = "#{include_path}/#{name}.rb"
  
     puts "*** loading data '#{name}' (#{path})..."
 
-    code = File.read( path )
-    
-    load_fixtures_worker( code )
-  end
-  
-  def load_fixtures_builtin( name ) # load from gem (built-in)
-    path = "#{WorldDB.root}/data/#{name}.rb"
+    CodeReader.new( logger, path ).eval( CodeContext )
 
-    puts "*** loading data '#{name}' (#{path})..."
+    # Prop.create!( :key => "db.#{name}.version", :value => WorldDB::VERSION )    
+  end
 
-    code = File.read( path )
-    
-    load_fixtures_worker( code )
+  def load_fixtures_builtin( name ) # load from gem (built-in)
+    load_fixtures_with_include_path( name, WorldDB.data_path )
   end
-  
 
-private
-  def load_fixtures_worker( code )
-    
-    self.class_eval( code )
 
-    # NB: same as
-    #
-    # module WorldDB
-    #   include WorldDB::Models
-    #  <code here>
-    # end
-    
-    # Prop.create!( :key => "db.#{name}.version", :value => SportDB::VERSION )
-    
-  end
-  
 end # class Loader
 end # module WorldDB

data/lib/worlddb/models/prop.rb

@@ -3,7 +3,33 @@
 module WorldDB::Models
 
   class Prop < ActiveRecord::Base
+
+    def self.create_from_worlddb_fixture!( name, path )
+      key   = "db.#{fixture_name_to_prop_key(name)}.version"
     
-  end  # class Prop
+      if path.starts_with?( WorldDB.data_path )
+        value = "world.yml.#{WorldDB::VERSION}"  # assume builtin
+      else
+        value = "file.txt.#{File.mtime(path).strftime('%Y.%m.%d')}"
+      end
+    
+      Prop.create!( key: key, value: value )
+    end
+    
+private
+
+  # helper
+  #   change at/2012_13/bl           to at.2012/13.bl
+  #    or    quali_2012_13_europe_c  to quali.2012/13.europe.c
   
+  def self.fixture_name_to_prop_key( name )
+    prop_key = name.gsub( '/', '.' )
+    prop_key = prop_key.gsub( /(\d{4})_(\d{2})/, '\1/\2' )  # 2012_13 => 2012/13
+    prop_key = prop_key.gsub( '_', '.' )
+    prop_key
+  end
+    
+    
+  end  # class Prop
+
 end # module WorldDB::Models

data/lib/worlddb/reader.rb

@@ -23,25 +23,33 @@ class Reader
     args.each do |arg|
       name = arg     # File.basename( arg, '.*' )
 
-      if opts.load?
-        load_countries_builtin( name )             if opts.countries?
-        load_regions_builtin( opts.country, name ) if opts.regions?
-        load_cities_builtin( opts.country, name )  if opts.cities?
+      data_path = opts.load? ? WorldDB.data_path : opts.data_path
+
+      if opts.countries?
+        load_countries_with_include_path( name, data_path )
+      elsif opts.regions?
+        load_regions_with_include_path( opts.country, name, data_path )
+      elsif opts.cities?
+        load_cities_with_include_path( opts.country, name, data_path )
       else
-        load_countries_with_include_path( name, opts.data_path )              if opts.countries?
-        load_regions_with_include_path( opts.country, name, opts.data_path )  if opts.regions?
-        load_cities_with_include_path( opts.country, name, opts.data_path )   if opts.cities?
+        ## todo: issue a warning here; no fixture type specified; assume country?
       end
-    end
+    end # each arg
 
   end
 
 
-  ############################
-  # load from file system
-  
   def load_with_include_path( name, include_path )
-    if name =~ /^([a-z]{3,})\/countries/
+
+    if name =~ /\/fifa/
+       load_xxx_with_include_path( 'fifa', name, include_path )
+    elsif name =~ /\/iso3/
+       load_xxx_with_include_path( 'iso3', name, include_path )
+    elsif name =~ /\/internet/
+       load_xxx_with_include_path( 'net', name, include_path )
+    elsif name =~ /\/motor/
+       load_xxx_with_include_path( 'motor', name, include_path )
+    elsif name =~ /^([a-z]{3,})\/countries/     # e.g. africa/countries or america/countries
       ## auto-add continent (from folder structure) as tag
       load_countries_with_include_path( name, include_path, :tags => $1 )
     elsif name =~ /\/([a-z]{2})\/cities/
@@ -56,71 +64,46 @@ class Reader
     end
   end
 
+  def load_builtin( name )  ## convenience helper (requires proper named files w/ convention)
+    load_with_include_path( name, WorldDB.data_path )
+  end
+
+
   def load_countries_with_include_path( name, include_path, more_values={} )
     load_fixtures_with_include_path_for( Country, name, include_path, more_values )
   end
 
-  def load_regions_with_include_path( country_key, name, include_path )
-    country = Country.find_by_key!( country_key )
-    puts "Country #{country.key} >#{country.title} (#{country.code})<"
-
-    load_fixtures_with_include_path_for( Region, name, include_path, country_id: country.id )
+  def load_countries_builtin( name, more_values={} )
+    load_countries_with_include_path( name, WorldDB.data_path, more_values )
   end
 
-  def load_cities_with_include_path( country_key, name, include_path )  
+
+  def load_regions_with_include_path( country_key, name, include_path )
     country = Country.find_by_key!( country_key )
     puts "Country #{country.key} >#{country.title} (#{country.code})<"
 
-    load_fixtures_with_include_path_for( City, name, include_path, country_id: country.id )
+    load_fixtures_with_include_path_for( Region, name, include_path, country_id: country.id )
   end
-  
-  ##################################
-  #  load from gem (built-in)
 
-  def load_builtin( name )  ## convenience helper (requires proper named files w/ convention)
-
-    if name =~ /\/fifa/
-       load_xxx_builtin( 'fifa', name )
-    elsif name =~ /\/iso3/
-       load_xxx_builtin( 'iso3', name )
-    elsif name =~ /\/internet/
-       load_xxx_builtin( 'net', name )
-    elsif name =~ /\/motor/
-       load_xxx_builtin( 'motor', name )
-    elsif name =~ /^([a-z]{3,})\/countries/     # e.g. africa/countries or america/countries
-      ## auto-add continent (from folder structure) as tag
-       load_countries_builtin( name, :tags => $1 )
-    elsif name =~ /\/([a-z]{2})\/cities/
-       load_cities_builtin( $1, name )
-    elsif name =~ /\/([a-z]{2})\/regions/
-       load_regions_builtin( $1, name )
-    else
-       puts "*** error: unknown world.db fixture type >#{name}<"
-       # todo/fix: exit w/ error
-    end
+  def load_regions_builtin( country_key, name )
+    load_regions_with_include_path( country_key, name, WorldDB.data_path )
   end
 
-  def load_countries_builtin( name, more_values )
-    load_fixtures_builtin_for( Country, name, more_values )
-  end
 
-  def load_regions_builtin( country_key, name )
+  def load_cities_with_include_path( country_key, name, include_path )
     country = Country.find_by_key!( country_key )
     puts "Country #{country.key} >#{country.title} (#{country.code})<"
-  
-    load_fixtures_builtin_for( Region, name, country_id: country.id )
+
+    load_fixtures_with_include_path_for( City, name, include_path, country_id: country.id )
   end
-  
-  def load_cities_builtin( country_key, name )
-    country = Country.find_by_key!( country_key )
-    puts "Country #{country.key} >#{country.title} (#{country.code})<"
 
-    load_fixtures_builtin_for( City, name, country_id: country.id )
+  def load_cities_builtin( country_key, name )
+    load_cities_with_include_path( country_key, name, WorldDB.data_path )
   end
 
 
-  def load_xxx_builtin( xxx, name )
-    path = "#{WorldDB.root}/data/#{name}.yml"
+  def load_xxx_with_include_path( xxx, name, include_path )
+    path = "#{include_path}/#{name}.yml"
 
     puts "*** parsing data '#{name}' (#{path})..."
 
@@ -132,7 +115,11 @@ class Reader
       country.save!
     end
 
-    Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "world.yml.#{WorldDB::VERSION}" )
+    Prop.create_from_worlddb_fixture!( name, path )
+  end
+
+  def load_xxx_builtin( xxx, name )
+    load_xxx_with_include_path( xxx, name, WorldDB.data_path )
   end
 
 
@@ -146,19 +133,7 @@ private
     
     load_fixtures_worker_for( clazz, reader )
 
-    Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "file.txt.#{File.mtime(path).strftime('%Y.%m.%d')}" )        
-  end
-
-  def load_fixtures_builtin_for( clazz, name, more_values={} ) # load from gem (built-in)
-    path = "#{WorldDB.root}/data/#{name}.txt"
-
-    puts "*** parsing data '#{name}' (#{path})..."
-
-    reader = ValuesReader.new( logger, path, more_values )
-    
-    load_fixtures_worker_for( clazz, reader )
-
-    Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "world.txt.#{WorldDB::VERSION}" )
+    Prop.create_from_worlddb_fixture!( name, path )
   end
 
 

data/lib/worlddb/readers/code_reader.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+class CodeReader
+
+  def initialize( logger=nil, path )
+    if logger.nil?
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::INFO
+    else
+      @logger = logger
+    end
+
+    @path  = path
+
+    ## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see worlddb/utils.rb
+    
+    @code  = File.read_utf8( @path )
+  end
+  
+  def eval( klass )
+    klass.class_eval( @code )
+
+    # NB: same as
+    #
+    # module WorldDB
+    #   include WorldDB::Models
+    #  <code here>
+    # end
+  end
+
+  attr_reader :logger
+
+end # class CodeReader

data/lib/worlddb/readers/hash_reader.rb

@@ -15,7 +15,15 @@ class HashReader
 
     ## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
     ## - see worlddb/utils.rb
-    @hash = YAML.load( File.read_utf8( @path ))
+    
+    text = File.read_utf8( @path )
+    ### hack for syck yaml parser (e.g.ruby 1.9.2) (cannot handle !!null)
+    ##   change it to !null to get plain nil
+    ##   w/ both syck and psych/libyml
+    
+    text = text.gsub( '!!null', '!null' )
+    
+    @hash = YAML.load( text )
   end
 
   attr_reader :logger

data/lib/worlddb/utils.rb

@@ -26,21 +26,6 @@ class File
   end
 end # class File
 
-
-
-  ##
-  # fix/todo: share helper w/ other readers
-  
-  # helper
-  #   change at/2012_13/bl           to at.2012/13.bl
-  #    or    quali_2012_13_europe_c  to quali.2012/13.europe.c
-  
-  def fixture_name_to_prop_key( name )
-    prop_key = name.gsub( '/', '.' )
-    prop_key = prop_key.gsub( /(\d{4})_(\d{2})/, '\1/\2' )  # 2012_13 => 2012/13
-    prop_key = prop_key.gsub( '_', '.' )
-    prop_key
-  end
   
   ############
   ### fix/todo: share helper for all text readers/parsers- where to put it?  

data/lib/worlddb/version.rb

@@ -1,5 +1,5 @@
 
 module WorldDB
-  VERSION = '0.6.8'
+  VERSION = '0.7.0'
 end
 

data/lib/worlddb.rb

@@ -30,6 +30,7 @@ require 'worlddb/models/tag'
 require 'worlddb/models/tagging'
 require 'worlddb/schema'       # NB: requires worlddb/models (include WorldDB::Models)
 require 'worlddb/utils'
+require 'worlddb/readers/code_reader'
 require 'worlddb/readers/line_reader'
 require 'worlddb/readers/values_reader'
 require 'worlddb/readers/hash_reader'
@@ -47,6 +48,11 @@ module WorldDB
   def self.root
     "#{File.expand_path( File.dirname(File.dirname(__FILE__)) )}"
   end
+  
+  # builtin path to fixture data
+  def self.data_path
+    "#{root}/data"
+  end
 
   def self.main
     Runner.new.run(ARGV)
@@ -67,82 +73,100 @@ module WorldDB
   end
 
   def self.fixtures  # all builtin fixtures; helper for covenience
+     africa_fixtures  +
+     america_fixtures +
+     europe_fixtures  +
+     asia_fixtures    +
+     oceania_fixtures
+  end
+
+  def self.africa_fixtures
+    ['countries',
+    '1_codes/fifa',
+    '1_codes/internet',
+    '1_codes/iso3'].map { |path| "africa/#{path}" }
+  end
+
+  def self.america_fixtures
+  ['countries',
+   '1_codes/fifa',
+   '1_codes/internet',
+   '1_codes/iso3',
+   '1_codes/motor',
+   'br/regions',
+   'br/cities',
+   'ca/regions',
+   'ca/cities',
+   'mx/regions',
+   'mx/cities',
+   'us/regions',
+   'us/cities',
+   've/regions',
+   've/cities'].map { |path| "america/#{path}" }
+  end
+
+  def self.asia_fixtures
+  ['countries',
+   '1_codes/fifa',
+   '1_codes/internet',
+   '1_codes/iso3',
+   'jp/cities'].map { |path| "asia/#{path}" }
+  end
+  
+  def self.europe_fixtures
+  ['countries',
+   '1_codes/fifa',
+   '1_codes/internet',
+   '1_codes/iso3',
+   '1_codes/motor',
+   'at/regions',
+   'at/cities',
+   'be/regions',
+   'be/cities',
+   'bg/cities',
+   'by/cities',
+   'ch/cities',
+   'cy/cities',
+   'cz/regions',
+   'cz/cities',
+   'de/regions',
+   'de/cities',
+   'dk/cities',
+   'ee/cities',
+   'en/regions',
+   'en/cities',
+   'es/regions',
+   'es/cities',
+   'fi/cities',
+   'fr/regions',
+   'fr/cities',
+   'gr/cities',
+   'hr/cities',
+   'hu/cities',
+   'ie/cities',
+   'it/cities',
+   'lt/cities',
+   'lv/cities',
+   'nl/cities',
+   'no/cities',
+   'pl/cities',
+   'pt/cities',
+   'ro/cities',
+   'rs/cities',
+   'ru/cities',
+   'sc/cities',
+   'se/cities',
+   'tr/cities',
+   'ua/cities',
+   'wa/cities'].map { |path| "europe/#{path}" }
+  end
 
-  ['africa/countries',
-   'africa/1_codes/fifa',
-   'africa/1_codes/internet',
-   'africa/1_codes/iso3',
-   'america/countries',
-   'america/1_codes/fifa',
-   'america/1_codes/internet',
-   'america/1_codes/iso3',
-   'america/1_codes/motor',
-   'america/br/regions',
-   'america/br/cities',
-   'america/ca/regions',
-   'america/ca/cities',
-   'america/mx/regions',
-   'america/mx/cities',
-   'america/us/regions',
-   'america/us/cities',
-   'america/ve/regions',
-   'america/ve/cities',
-   'asia/countries',
-   'asia/1_codes/fifa',
-   'asia/1_codes/internet',
-   'asia/1_codes/iso3',
-   'asia/jp/cities',
-   'europe/countries',
-   'europe/1_codes/fifa',
-   'europe/1_codes/internet',
-   'europe/1_codes/iso3',
-   'europe/1_codes/motor',
-   'europe/at/regions',
-   'europe/at/cities',
-   'europe/be/regions',
-   'europe/be/cities',
-   'europe/bg/cities',
-   'europe/by/cities',
-   'europe/ch/cities',
-   'europe/cy/cities',
-   'europe/cz/regions',
-   'europe/cz/cities',
-   'europe/de/regions',
-   'europe/de/cities',
-   'europe/dk/cities',
-   'europe/ee/cities',
-   'europe/en/regions',
-   'europe/en/cities',
-   'europe/es/regions',
-   'europe/es/cities',
-   'europe/fi/cities',
-   'europe/fr/regions',
-   'europe/fr/cities',
-   'europe/gr/cities',
-   'europe/hr/cities',
-   'europe/hu/cities',
-   'europe/ie/cities',
-   'europe/it/cities',
-   'europe/lt/cities',
-   'europe/lv/cities',
-   'europe/nl/cities',
-   'europe/no/cities',
-   'europe/pl/cities',
-   'europe/pt/cities',
-   'europe/ro/cities',
-   'europe/rs/cities',
-   'europe/ru/cities',
-   'europe/sc/cities',
-   'europe/se/cities',
-   'europe/tr/cities',
-   'europe/ua/cities',
-   'europe/wa/cities',
-   'oceania/countries',
-   'oceania/1_codes/fifa',
-   'oceania/1_codes/internet',
-   'oceania/1_codes/iso3',
-   'oceania/au/cities'
-   ]
+  def self.oceania_fixtures
+   ['countries',
+   '1_codes/fifa',
+   '1_codes/internet',
+   '1_codes/iso3',
+   'au/cities'].map { |path| "oceania/#{path}" }
   end
 
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: worlddb
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 3
   prerelease: 
   segments: 
   - 0
-  - 6
-  - 8
-  version: 0.6.8
+  - 7
+  - 0
+  version: 0.7.0
 platform: ruby
 authors: 
 - Gerald Bauer
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-12-02 00:00:00 Z
+date: 2012-12-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activerecord
@@ -175,6 +175,7 @@ files:
 - lib/worlddb/models/tag.rb
 - lib/worlddb/models/tagging.rb
 - lib/worlddb/reader.rb
+- lib/worlddb/readers/code_reader.rb
 - lib/worlddb/readers/hash_reader.rb
 - lib/worlddb/readers/line_reader.rb
 - lib/worlddb/readers/values_reader.rb