worlddb 2.0.4 → 2.0.5

data/Manifest.txt

@@ -27,7 +27,8 @@ lib/worlddb/models/tagdb/tagging.rb
 lib/worlddb/models/usage.rb
 lib/worlddb/patterns.rb
 lib/worlddb/reader.rb
-lib/worlddb/readers/base.rb
+lib/worlddb/reader_file.rb
+lib/worlddb/reader_zip.rb
 lib/worlddb/readers/city.rb
 lib/worlddb/readers/country.rb
 lib/worlddb/readers/lang.rb

data/README.md

@@ -2,8 +2,8 @@
 
 world.db Command Line Tool in Ruby
 
-* home  :: [github.com/geraldb/world.db.ruby](https://github.com/geraldb/world.db.ruby)
-* bugs  :: [github.com/geraldb/world.db.ruby/issues](https://github.com/geraldb/world.db.ruby/issues)
+* home  :: [github.com/worlddb/world.db.ruby](https://github.com/worlddb/world.db.ruby)
+* bugs  :: [github.com/worlddb/world.db.ruby/issues](https://github.com/worlddb/world.db.ruby/issues)
 * gem   :: [rubygems.org/gems/worlddb](https://rubygems.org/gems/worlddb)
 * rdoc  :: [rubydoc.info/gems/worlddb](http://rubydoc.info/gems/worlddb)
 * forum :: [groups.google.com/group/openmundi](https://groups.google.com/group/openmundi)

data/Rakefile

@@ -5,23 +5,24 @@ require './lib/worlddb/version.rb'
 Hoe.spec 'worlddb' do
   
   self.version = WorldDb::VERSION
-  
+
   self.summary = "worlddb - world.db command line tool"
   self.description = summary
 
-  self.urls    = ['https://github.com/geraldb/world.db.ruby']
-  
+  self.urls    = ['https://github.com/worlddb/world.db.ruby']
+
   self.author  = 'Gerald Bauer'
   self.email   = 'openmundi@googlegroups.com'
 
   self.extra_deps = [
     ['props'],         # settings / prop(ertie)s / env / INI
     ['logutils'],     # logging
-    ['textutils', '>= 0.9.4'], # e.g.  >= 0.6 && <= 1.0  ## will include logutils, props
+    ['textutils', '>= 0.9.9'], # e.g.  >= 0.6 && <= 1.0  ## will include logutils, props
     ['tagutils'],     # tags n categories for activerecord
 
      ## 3rd party
     ['gli', '>= 2.9'],
+    ['rubyzip'],       ## todo: pull in via textutils ??
     ['activerecord']  # NB: will include activesupport,etc.
     ### ['sqlite3',      '~> 1.3']  # NB: install on your own; remove dependency
   ]

data/lib/worlddb.rb

@@ -5,15 +5,20 @@
 
 # core and stlibs
 
-require 'yaml'
 require 'pp'
 require 'logger'
 require 'optparse'
 require 'fileutils'
+require 'uri'
 require 'erb'
+require 'json'
+require 'yaml'
+
 
 # 3rd party gems / libs
 
+require 'zip'
+
 require 'active_record'   ## todo: add sqlite3? etc.
 
 require 'logutils'
@@ -52,7 +57,6 @@ require 'worlddb/models/tagdb/tagging'
 require 'worlddb/schema'       # NB: requires worlddb/models (include WorldDB::Models)
 require 'worlddb/matcher'
 
-require 'worlddb/readers/base'
 require 'worlddb/readers/lang'
 require 'worlddb/readers/usage'
 require 'worlddb/readers/country'
@@ -60,6 +64,8 @@ require 'worlddb/readers/region'
 require 'worlddb/readers/city'
 
 require 'worlddb/reader'
+require 'worlddb/reader_file'
+require 'worlddb/reader_zip'
 require 'worlddb/deleter'
 require 'worlddb/stats'
 require 'worlddb/stats_comp'
@@ -67,14 +73,6 @@ require 'worlddb/stats_comp'
 
 module WorldDb
 
-  def self.banner
-    "worlddb/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
-  end
-
-  def self.root
-    "#{File.expand_path( File.dirname(File.dirname(__FILE__)) )}"
-  end
-
   def self.main
     require 'worlddb/cli/main'
     ## Runner.new.run(ARGV) - old code
@@ -99,6 +97,12 @@ module WorldDb
     reader.load_setup( setup )
   end
 
+  def self.read_setup_from_zip( zip_name, setup, include_path, opts={} ) ## todo/check - use a better (shorter) name ??
+    reader = ZipReader.new( zip_name, include_path, opts )
+    reader.load_setup( setup )
+    reader.close
+  end
+
   def self.read_all( include_path, opts={} )  # load all builtins (using plain text reader); helper for convenience
     read_setup( 'setups/all', include_path, opts )
   end # method read_all
@@ -129,14 +133,9 @@ module WorldDb
 
 end  # module WorldDb
 
-###########################################
-# fix: remove old alias for WorldDb ??
-WorldDB = WorldDb
-
 
 if __FILE__ == $0
   WorldDb.main
 else
-  # say hello
-  puts WorldDb.banner
-end
+  puts WorldDb.banner    # say hello
+end

data/lib/worlddb/matcher.rb

@@ -97,20 +97,31 @@ module Matcher
   end
 
   def match_regions_for_country( name, &blk )
-    ## todo: check if there's a better (more ruby way) to pass along code block ??
-    match_xxx_for_country( name, 'regions', &blk )
+    ## also try synonyms e.g. old regions (if not match for states)
+    found = match_xxx_for_country( name, 'states', &blk )
+    found = match_xxx_for_country( name, 'regions', &blk ) unless found
+    found
   end
 
-  def match_regions_abbr_for_country( name, &blk )
-    match_xxx_for_country( name, 'regions\.abbr', &blk )  # NB: . gets escaped for regex, that is, \.
+  def match_regions_abbr_for_country( name, &blk )  # NB: . gets escaped for regex, that is, \.
+    ## also try synonyms e.g. old regions (if not match for states)
+    found = match_xxx_for_country( name, 'states\.abbr', &blk )
+    found = match_xxx_for_country( name, 'regions\.abbr', &blk ) unless found
+    found
   end
 
   def match_regions_iso_for_country( name, &blk )  # NB: . gets escaped for regex, that is, \.
-    match_xxx_for_country( name, 'regions\.iso', &blk )
+    ## also try synonyms e.g. old regions (if not match for states)
+    found = match_xxx_for_country( name, 'states\.iso', &blk )
+    found = match_xxx_for_country( name, 'regions\.iso', &blk ) unless found
+    found 
   end
 
   def match_regions_nuts_for_country( name, &blk )  # NB: . gets escaped for regex, that is, \.
-    match_xxx_for_country( name, 'regions\.nuts', &blk )
+    ## also try synonyms e.g. old regions (if not match for states)
+    found = match_xxx_for_country( name, 'states\.nuts', &blk )
+    found = match_xxx_for_country( name, 'regions\.nuts', &blk ) unless found
+    found
   end
 
 

data/lib/worlddb/reader.rb

@@ -3,7 +3,7 @@
 module WorldDb
 
 
-class Reader
+class ReaderBase
 
   include LogUtils::Logging
 
@@ -14,19 +14,13 @@ class Reader
 
 ## value helpers e.g. is_year?, is_taglist? etc.
   include TextUtils::ValueHelper
-  
 
 
-  attr_reader :include_path
-
   def skip_tags?()   @skip_tags == true;  end
   def strict?()      @strict == true;     end
 
 
-  def initialize( include_path, opts = {} )
-    
-    @include_path = include_path
-    
+  def initialize( opts={} )
     ## option: do NOT generate/add any tags for countries/regions/cities
     @skip_tags =  opts[:skip_tags].present? ? true : false
     ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
@@ -35,11 +29,7 @@ class Reader
 
 
   def load_setup( name )
-    path = "#{include_path}/#{name}.txt"
-
-    logger.info "parsing data '#{name}' (#{path})..."
-
-    reader = FixtureReader.new( path )
+    reader = create_fixture_reader( name )
 
     reader.each do |fixture|
       load( fixture )
@@ -56,11 +46,13 @@ class Reader
     elsif name =~ /^lang/
        ## todo: pass along opts too
        ## use match_usage( name ) - why? why not?? ???
-       LangReader.new( include_path ).read( name )
+       r = create_lang_reader( name )
+       r.read()
     elsif name =~ /\/lang/
        ## todo: pass along opts too
        ## use match_usage( name ) - why? why not?? ???
-       UsageReader.new( include_path ).read( name )
+       r = create_usage_reader( name )
+       r.read()
     elsif name =~ /\/fifa/
        load_xxx( 'fifa', name )
     elsif name =~ /\/iso3/
@@ -72,22 +64,27 @@ class Reader
     elsif name =~ /^tag.*\.\d$/
        ## todo: pass along opts too
        ## use match_tags( name ) - why? why not?? ???
-       TagDb::TagReader.new( include_path ).read( name )
+       
+       ######## FIX: add back again
+       ### fix: use read() only, that is, w/o name
+       ## r = create_tag_reader( name )
+       ## r.read()
     elsif match_countries_for_continent( name ) do |continent|  # # e.g. africa/countries or america/countries
             ### NB: continent changed to regions (e.g. middle-east, caribbean, north-america, etc.)
             ## auto-add continent (from folder structure) as tag
             ## fix: allow dash/hyphen/minus in tag
 
-            r = CountryReader.new( include_path )
-            r.read( name, tags: continent.tr('-', '_') )
+            ### todo/fix: add opts - how??
+            r = create_country_reader( name, tags: continent.tr('-', '_') )
+            r.read()
           end
     elsif match_cities_for_country( name ) do |country_key|  #  name =~ /\/([a-z]{2})\/cities/
             ## auto-add required country code (from folder structure)
             country = Country.find_by_key!( country_key )
             logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
 
-            r = CityReader.new( include_path )
-            r.read( name, country_id: country.id )
+            r = create_city_reader( name, country_id: country.id )
+            r.read()
           end
     elsif match_regions_abbr_for_country( name ) do |country_key|   # name =~ /\/([a-z]{2})\/regions\.abbr/
             load_regions_xxx( country_key, 'abbr', name )
@@ -103,8 +100,8 @@ class Reader
             country = Country.find_by_key!( country_key )
             logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
 
-            r = RegionReader.new( include_path )
-            r.read( name, country_id: country.id )
+            r = create_region_reader( name, country_id: country.id )
+            r.read()
           end
     else
       logger.error "unknown world.db fixture type >#{name}<"
@@ -118,7 +115,7 @@ class Reader
     country = Country.find_by_key!( country_key )
     logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
 
-    reader = HashReaderV2.new( name, include_path )
+    reader = create_hash_reader( name )
 
     reader.each do |key, value|
       region = Region.find_by_country_id_and_key!( country.id, key )
@@ -130,7 +127,7 @@ class Reader
 
   ### use ContinentRefReader
   def load_continent_refs( name )
-    reader = HashReaderV2.new( name, include_path )
+    reader = create_hash_reader( name )
 
     reader.each do |key, value|
       country = Country.find_by_key!( key )
@@ -142,7 +139,7 @@ class Reader
 
   ### use ContinentDef Reader
   def load_continent_defs( name, more_attribs={} )
-    reader = ValuesReaderV2.new( name, include_path, more_attribs )
+    reader = create_values_reader( name, more_attribs )
 
     reader.each_line do |attribs, values|
 
@@ -168,7 +165,7 @@ class Reader
 
   ### use CountryAttr Reader
   def load_xxx( xxx, name )
-    reader = HashReaderV2.new( name, include_path )
+    reader = create_hash_reader( name )
 
     reader.each do |key, value|
       country = Country.find_by_key!( key )
@@ -177,5 +174,5 @@ class Reader
     end
   end
 
-end # class Reader
+end # class ReaderBase
 end # module WorldDb

data/lib/worlddb/reader_file.rb

@@ -0,0 +1,80 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class Reader < ReaderBase
+
+  def initialize( include_path, opts={} )
+    super( opts )
+
+    @include_path = include_path
+  end
+
+
+  def create_fixture_reader( name )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (setup) '#{name}' (#{path})..."
+
+    FixtureReader.from_file( path )
+  end
+
+  def create_lang_reader( name )
+    path = "#{@include_path}/#{name}.yml"   ## hash reader - use .yml??
+    logger.info "parsing data (lang) '#{name}' (#{path})..."
+
+    LangReader.from_file( path )
+  end
+
+  def create_usage_reader( name )
+    path = "#{@include_path}/#{name}.yml"   ## hash reader - use .yml??
+    logger.info "parsing data (usage) '#{name}' (#{path})..."
+
+    UsageReader.from_file( path )
+  end
+
+
+  def create_country_reader( name, more_attribs={} )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (country) '#{name}' (#{path})..."
+
+    CountryReader.from_file( path, more_attribs )
+  end
+
+  def create_region_reader( name, more_attribs={} )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (region) '#{name}' (#{path})..."
+
+    RegionReader.from_file( path, more_attribs )
+  end
+
+  def create_city_reader( name, more_attribs={} )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (city) '#{name}' (#{path})..."
+
+    CityReader.from_file( path, more_attribs )
+  end
+
+
+  def create_hash_reader( name )
+    path = "#{@include_path}/#{name}.yml"
+    logger.info "parsing data (hash) '#{name}' (#{path})..."
+
+    HashReader.from_file( path )
+  end
+
+  def create_values_reader( name, more_attribs={} )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (values) '#{name}' (#{path})..."
+
+    ValuesReader.from_file( path, more_attribs )
+  end
+
+
+#  def create_tag_reader( name )
+#    ## fix: change to new from_file() style
+#    TagDb::TagReader.new( @include_path )
+#  end
+
+
+end # class Reader
+end # module WorldDb

data/lib/worlddb/reader_zip.rb

@@ -0,0 +1,154 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class ZipReader < ReaderBase
+
+
+  def initialize( name, include_path, opts = {} )
+    super( opts )
+
+    ## todo/fix: make include_path an opts (included in opts?) - why? why not??
+    path = "#{include_path}/#{name}.zip"
+    
+    ## todo: check if zip exists
+    @zip_file = Zip::File.open( path ) ## NOTE: do NOT create if file is missing; let it crash
+
+    ### allow prefix (path) in name
+    ### e.g. assume all files relative to setup manifest
+    ## e.g. at-austria-master/setups/all.txt or
+    ## be-belgium-master/setups/all.txt
+    ## for
+    ## setups/all.txt
+    ###
+    ## will get (re)set w/ fixture/setup reader
+    ##
+    ## todo/fix: change/rename to @relative_path ?? - why? why not?
+    @zip_prefix = ''
+  end
+
+  def close
+    ## todo/check: add a close method - why? why not ???
+    @zip_file.close
+  end
+
+
+
+  def create_fixture_reader( name )
+    ## e.g. pass in => setups/all or setups/test etc. e.g. w/o .txt extension
+    query = "**/#{name}.txt"
+
+    ## note: returns an array of Zip::Entry
+    candidates = @zip_file.glob( query )
+    pp candidates
+
+    ## use first candidates entry as match
+    ## todo/fix: issue warning if more than one entries/matches!!
+
+    ## get fullpath e.g. at-austria-master/setups/all.txt
+    path = candidates[0].name
+    logger.debug "  zip entry path >>#{path}<<"
+
+    ## cut-off at-austria-master/    NOTE: includes trailing slash (if present)
+    ## logger.debug "  path.size #{path.size} >>#{path}<<"
+    ## logger.debug "  name.size #{name.size+4} >>#{name}<<"
+
+    ## note: add +4 for extension (.txt)
+    @zip_prefix = path[ 0...(path.size-(name.size+4)) ]
+    logger.debug "  zip entry prefix >>#{@zip_prefix}<<"
+
+    logger.info "parsing data (setup) in zip '#{name}' (#{path})..."
+
+    FixtureReader.from_zip( @zip_file, path )
+  end
+
+
+  def create_lang_reader( name )
+    path = name_to_zip_entry_path( name, '.yml' )  ## hash reader - use .yml??
+    logger.info "parsing data (lang) in zip '#{name}' (#{path})..."
+
+    LangReader.from_zip( @zip_file, path )
+  end
+
+  def create_usage_reader( name )
+    path = name_to_zip_entry_path( name, '.yml' )  ## hash reader - use .yml??
+    logger.info "parsing data (usage) in zip '#{name}' (#{path})..."
+
+    UsageReader.from_zip( @zip_file, path )
+  end
+
+
+  def create_country_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (country) in zip '#{name}' (#{path})..."
+
+    CountryReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_region_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (region) in zip '#{name}' (#{path})..."
+
+    RegionReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_city_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (city) in zip '#{name}' (#{path})..."
+
+    CityReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+
+  def create_hash_reader( name )
+    path = name_to_zip_entry_path( name, '.yml' )  ## hash reader - use .yml??
+    logger.info "parsing data (hash) in zip '#{name}' (#{path})..."
+
+    HashReader.from_zip( @zip_file, path )
+  end
+
+  def create_values_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (values) in zip '#{name}' (#{path})..."
+
+    ValuesReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+
+#  def create_tag_reader( name )
+#    ## fix: change to new from_file() style
+#    TagDb::TagReader.new( @include_path )
+#  end
+
+private
+
+  def path_to_real_path( path )
+    # map name to name_real_path
+    # name might include !/ for virtual path (gets cut off)
+    # e.g. at-austria!/w-wien/beers becomse w-wien/beers
+    pos = path.index( '!/')
+    if pos.nil?
+      path # not found; real path is the same as name
+    else
+      # cut off everything until !/ e.g.
+      # at-austria!/w-wien/beers becomes
+      # w-wien/beers
+      path[ (pos+2)..-1 ]
+    end
+  end
+
+  def name_to_zip_entry_path( name, extension='.txt' )
+    path = "#{name}#{extension}"
+
+    real_path = path_to_real_path( path )
+
+    # NOTE: add possible zip entry prefix path
+    #          (if present includes trailing slash e.g. /)
+    entry_path = "#{@zip_prefix}#{real_path}"
+    entry_path
+  end
+
+
+
+end # class ZipReader
+end # module WorldDb

data/lib/worlddb/readers/city.rb

@@ -2,10 +2,73 @@
 
 module WorldDb
 
-class CityReader < BaseReader
+class CityReader
 
-  def read( name, more_attribs={} )
-    reader = ValuesReaderV2.new( name, include_path, more_attribs )
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  def self.from_zip( zip_file, entry_path, more_attribs={} )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text, more_attribs )
+  end
+
+
+  def self.from_file( path, more_attribs={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_string( text, more_attribs={} )
+    CityReader.new( text, more_attribs )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, more_attribs={} )
+    ## todo/fix: how to add opts={} ???
+
+    @text         = text
+    @more_attribs = more_attribs
+  end
+
+
+  def read()
+    reader = ValuesReader.from_string( @text, @more_attribs )
 
     reader.each_line do |attribs, values|
       opts = { skip_tags: skip_tags? }

data/lib/worlddb/readers/country.rb

@@ -2,10 +2,71 @@
 
 module WorldDb
 
-class CountryReader < BaseReader
+class CountryReader
 
-  def read( name, more_attribs={} )
-    reader = ValuesReaderV2.new( name, include_path, more_attribs )
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  def self.from_zip( zip_file, entry_path, more_attribs={} )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_file( path, more_attribs={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, more_attribs )    
+  end
+
+  def self.from_string( text, more_attribs={} )
+    CountryReader.new( text, more_attribs )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, more_attribs={} )
+    ## todo/fix: how to add opts={} ???
+
+    @text         = text
+    @more_attribs = more_attribs
+  end
+
+  def read()
+    reader = ValuesReader.from_string( @text, @more_attribs )
 
     reader.each_line do |attribs, values|
       opts = { skip_tags: skip_tags? }

data/lib/worlddb/readers/lang.rb

@@ -2,10 +2,74 @@
 
 module WorldDb
 
-class LangReader < BaseReader
+class LangReader
 
-  def read( name ) 
-    reader = HashReaderV2.new( name, include_path )
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  ## todo: add opts={} etc.
+  def self.from_zip( zip_file, entry_path )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text )
+  end
+
+  def self.from_file( path, opts={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, opts )
+  end
+
+  def self.from_string( text, opts={} )
+    LangReader.new( text, opts )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, opts={} )
+    @text = text
+
+    ## option: do NOT generate/add any tags for countries/regions/cities
+    @skip_tags =  opts[:skip_tags].present? ? true : false
+    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
+    @strict    =  opts[:strict].present? ? true : false
+  end
+
+  def read()
+    reader = HashReader.from_string( @text )
 
     reader.each do |key, value|
 

data/lib/worlddb/readers/region.rb

@@ -2,10 +2,72 @@
 
 module WorldDb
 
-class RegionReader < BaseReader
+class RegionReader
 
-  def read( name, more_attribs={} )
-    reader = ValuesReaderV2.new( name, include_path, more_attribs )
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  def self.from_zip( zip_file, entry_path, more_attribs={} )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_file( path, more_attribs={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_string( text, more_attribs={} )
+    RegionReader.new( text, more_attribs )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, more_attribs={} )
+    ## todo/fix: how to add opts={} ???
+
+    @text         = text
+    @more_attribs = more_attribs
+  end
+
+
+  def read()
+    reader = ValuesReader.from_string( @text, @more_attribs )
 
     reader.each_line do |attribs, values|
       opts = { skip_tags: skip_tags? }

data/lib/worlddb/readers/usage.rb

@@ -2,10 +2,73 @@
 
 module WorldDb
 
-class UsageReader < BaseReader
+class UsageReader
 
-  def read( name )
-    reader = HashReaderV2.new( name, include_path )
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+  ## todo: add opts
+  def self.from_zip( zip_file, entry_path )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text )
+  end
+
+  def self.from_file( path, opts={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, opts )
+  end
+
+  def self.from_string( text, opts={} )
+    UsageReader.new( text, opts )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, opts={} )
+    @text = text
+
+    ## option: do NOT generate/add any tags for countries/regions/cities
+    @skip_tags =  opts[:skip_tags].present? ? true : false
+    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
+    @strict    =  opts[:strict].present? ? true : false
+  end
+
+  def read()
+    reader = HashReader.from_string( @text )
 
     reader.each do |key, value|
       

data/lib/worlddb/version.rb

@@ -1,6 +1,23 @@
 
 module WorldDb
-  VERSION = '2.0.4'  # sync version w/ sport.db - why? why not?
-end
 
+  # sync version w/ sport.db n friends - why? why not?
+  MAJOR = 2 ## todo: namespace inside version or something - why? why not??
+  MINOR = 0
+  PATCH = 5
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "worlddb/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}"
+  end
+
+end
 

data/test/helper.rb

@@ -6,18 +6,6 @@
 # require 'minitest/unit'
 require 'minitest/autorun'
 
-# include MiniTest::Unit  # lets us use TestCase instead of MiniTest::Unit::TestCase
-
-
-# ruby stdlibs
-
-require 'json'
-require 'uri'
-require 'pp'
-
-# ruby gems
-
-require 'active_record'
 
 # our own code
 

data/test/test_fixture_matchers.rb

@@ -3,7 +3,7 @@
 require 'helper'
 
 
-class TestFixtureMatchers < MiniTest::Unit::TestCase
+class TestFixtureMatchers < MiniTest::Test
 
   include WorldDb::Matcher
 

data/test/test_model_city.rb

@@ -3,7 +3,7 @@
 
 require 'helper'
 
-class TestModelCity < MiniTest::Unit::TestCase
+class TestModelCity < MiniTest::Test
 
   def setup
     #  delete all countries, regions, cities in in-memory only db

data/test/test_model_comp.rb

@@ -3,7 +3,7 @@
 
 require 'helper'
 
-class TestModelComp < MiniTest::Unit::TestCase
+class TestModelComp < MiniTest::Test
 
   def setup
     #  delete all countries, regions, cities in in-memory only db

data/test/test_model_country.rb

@@ -2,7 +2,7 @@
 
 require 'helper'
 
-class TestModelCountry < MiniTest::Unit::TestCase
+class TestModelCountry < MiniTest::Test
 
   def setup
     #  delete all countries, regions, cities in in-memory only db

data/test/test_model_region.rb

@@ -3,7 +3,7 @@
 
 require 'helper'
 
-class TestModelRegion < MiniTest::Unit::TestCase
+class TestModelRegion < MiniTest::Test
 
   def setup
     #  delete all countries, regions, cities in in-memory only db

data/test/test_models.rb

@@ -3,7 +3,7 @@
 
 require 'helper'
 
-class TestModels < MiniTest::Unit::TestCase
+class TestModels < MiniTest::Test
 
   def setup
     #  delete all countries, regions, cities in in-memory only db

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: worlddb
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-15 00:00:00.000000000 Z
+date: 2014-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: props
-  requirement: &85578640 !ruby/object:Gem::Requirement
+  requirement: &75136710 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *85578640
+  version_requirements: *75136710
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &85576130 !ruby/object:Gem::Requirement
+  requirement: &75136360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,21 +32,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *85576130
+  version_requirements: *75136360
 - !ruby/object:Gem::Dependency
   name: textutils
-  requirement: &85575800 !ruby/object:Gem::Requirement
+  requirement: &75135990 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.9.4
+        version: 0.9.9
   type: :runtime
   prerelease: false
-  version_requirements: *85575800
+  version_requirements: *75135990
 - !ruby/object:Gem::Dependency
   name: tagutils
-  requirement: &85575530 !ruby/object:Gem::Requirement
+  requirement: &75135720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *85575530
+  version_requirements: *75135720
 - !ruby/object:Gem::Dependency
   name: gli
-  requirement: &85574960 !ruby/object:Gem::Requirement
+  requirement: &75135460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,21 @@ dependencies:
         version: '2.9'
   type: :runtime
   prerelease: false
-  version_requirements: *85574960
+  version_requirements: *75135460
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: &75135230 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *75135230
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &85574780 !ruby/object:Gem::Requirement
+  requirement: &75134990 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *85574780
+  version_requirements: *75134990
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &85574380 !ruby/object:Gem::Requirement
+  requirement: &75132620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,18 +98,18 @@ dependencies:
         version: '4.0'
   type: :development
   prerelease: false
-  version_requirements: *85574380
+  version_requirements: *75132620
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &85573790 !ruby/object:Gem::Requirement
+  requirement: &75132200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.11'
+        version: '3.13'
   type: :development
   prerelease: false
-  version_requirements: *85573790
+  version_requirements: *75132200
 description: worlddb - world.db command line tool
 email: openmundi@googlegroups.com
 executables:
@@ -138,7 +149,8 @@ files:
 - lib/worlddb/models/usage.rb
 - lib/worlddb/patterns.rb
 - lib/worlddb/reader.rb
-- lib/worlddb/readers/base.rb
+- lib/worlddb/reader_file.rb
+- lib/worlddb/reader_zip.rb
 - lib/worlddb/readers/city.rb
 - lib/worlddb/readers/country.rb
 - lib/worlddb/readers/lang.rb
@@ -156,7 +168,7 @@ files:
 - test/test_model_region.rb
 - test/test_models.rb
 - .gemtest
-homepage: https://github.com/geraldb/world.db.ruby
+homepage: https://github.com/worlddb/world.db.ruby
 licenses:
 - Public Domain
 post_install_message: 

data/lib/worlddb/readers/base.rb

@@ -1,41 +0,0 @@
-# encoding: UTF-8
-
-module WorldDb
-
-
-class BaseReader
-
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-
-  attr_reader :include_path
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-
-  def initialize( include_path, opts = {} )
-    
-    @include_path = include_path
-    
-    ## option: do NOT generate/add any tags for countries/regions/cities
-    @skip_tags =  opts[:skip_tags].present? ? true : false
-    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
-    @strict    =  opts[:strict].present? ? true : false
-  end
-
-
-  def read( name, more_attribs={} )
-    puts "error: overwrite in concrete reader class!!!"  ### overwrite!!!!
-  end
-
-
-end # class BaseReader
-end # module WorldDb