worlddb-models 2.1.0

data/lib/worlddb/reader.rb

@@ -0,0 +1,224 @@
+# encoding: UTF-8
+
+module WorldDb
+
+
+class ReaderBase
+
+  include LogUtils::Logging
+
+## make models available in sportdb module by default with namespace
+#  e.g. lets you use City instead of Models::City
+  include Models
+  include Matcher  # e.g. match_cities_for_country, match_regions_for_country, etc.
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+
+  def initialize( opts={} )
+    ## option: do NOT generate/add any tags for countries/regions/cities
+    @skip_tags =  opts[:skip_tags].present? ? true : false
+    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
+    @strict    =  opts[:strict].present? ? true : false
+  end
+
+
+  def load_setup( name )
+    reader = create_fixture_reader( name )
+
+    reader.each do |fixture|
+      load( fixture )
+    end
+  end # method load_setup
+
+
+  def load( name )
+
+    if name =~ /^continents/
+       load_continent_defs( name )
+    elsif name =~ /\/continents/
+       load_continent_refs( name )
+    elsif name =~ /^lang/
+       ## todo: pass along opts too
+       ## use match_usage( name ) - why? why not?? ???
+       r = create_lang_reader( name )
+       r.read()
+    elsif name =~ /\/lang/
+       ## todo: pass along opts too
+       ## use match_usage( name ) - why? why not?? ???
+       r = create_usage_reader( name )
+       r.read()
+    elsif name =~ /\/fifa/     ||
+          name =~ /\/fips/     ||
+          name =~ /\/internet/ ||
+          name =~ /\/ioc/      ||
+          name =~ /\/iso/      ||
+          name =~ /\/motor/
+       load_codes( name )
+    elsif name =~ /^tag.*\.\d$/
+       ## todo: pass along opts too
+       ## use match_tags( name ) - why? why not?? ???
+       
+       ######## FIX: add back again
+       ### fix: use read() only, that is, w/o name
+       ## r = create_tag_reader( name )
+       ## r.read()
+    elsif match_countries_for_continent( name ) do |continent|  # # e.g. africa/countries or america/countries
+            ### NB: continent changed to regions (e.g. middle-east, caribbean, north-america, etc.)
+            ## auto-add continent (from folder structure) as tag
+            ## fix: allow dash/hyphen/minus in tag
+
+            ### todo/fix: add opts - how??
+            r = create_country_reader( name, tags: continent.tr('-', '_') )
+            r.read()
+          end
+    elsif match_cities_for_country( name ) do |country_key|  #  name =~ /\/([a-z]{2})\/cities/
+            ## auto-add required country code (from folder structure)
+            country = Country.find_by_key!( country_key )
+            logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
+
+            r = create_city_reader( name, country_id: country.id )
+            r.read()
+          end
+    elsif match_regions_abbr_for_country( name ) do |country_key|   # name =~ /\/([a-z]{2})\/regions\.abbr/
+            load_regions_xxx( country_key, 'abbr', name )
+          end
+    elsif match_regions_iso_for_country( name ) do |country_key|  # name =~ /\/([a-z]{2})\/regions\.iso/
+            load_regions_xxx( country_key, 'iso', name )
+          end
+    elsif match_regions_nuts_for_country( name ) do |country_key|  # name =~ /\/([a-z]{2})\/regions\.nuts/
+            load_regions_xxx( country_key, 'nuts', name )
+          end
+    elsif match_regions_for_country( name ) do |country_key|  # name =~ /\/([a-z]{2})\/regions/
+            ## auto-add required country code (from folder structure)
+            country = Country.find_by_key!( country_key )
+            logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
+
+            r = create_region_reader( name, country_id: country.id )
+            r.read()
+          end
+    else
+      logger.error "unknown world.db fixture type >#{name}<"
+      # todo/fix: exit w/ error
+    end
+  end
+
+
+  ### use RegionAttrReader
+  def load_regions_xxx( country_key, xxx, name )
+    country = Country.find_by_key!( country_key )
+    logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
+
+    reader = create_hash_reader( name )
+
+    reader.each do |key, value|
+      region = Region.find_by_country_id_and_key!( country.id, key )
+      region.send( "#{xxx}=", value )
+      region.save!
+    end
+  end
+
+
+  ### use ContinentRefReader
+  def load_continent_refs( name )
+    reader = create_hash_reader( name )
+
+    reader.each do |key, value|
+      country = Country.find_by_key!( key )
+      continent = Continent.find_by_key!( value )
+      country.continent_id = continent.id
+      country.save!
+    end
+  end
+
+  ### use ContinentDef Reader
+  def load_continent_defs( name, more_attribs={} )
+    reader = create_values_reader( name, more_attribs )
+
+    reader.each_line do |attribs, values|
+
+      ## check optional values
+      values.each_with_index do |value, index|
+        logger.warn "unknown type for value >#{value}<"
+      end
+
+      rec = Continent.find_by_key( attribs[ :key ] )
+      if rec.present?
+        logger.debug "update Continent #{rec.id}-#{rec.key}:"
+      else
+        logger.debug "create Continent:"
+        rec = Continent.new
+      end
+
+      logger.debug attribs.to_json
+
+      rec.update_attributes!( attribs )
+
+    end # each lines
+  end # load_continent_defs
+
+
+  def load_codes( name )
+    reader = create_line_reader( name )
+
+    reader.each_line do |line|
+
+      values = line.split(',')
+
+      ## logger.debug '[>' + values.join( '<|>' ) + '<]'
+
+      if name =~ /iso/
+        # special case for iso
+        #  country ref, alpha2, alpha3, num
+        country_name = values[0].strip
+      else
+        #  code, country ref
+        country_name = values[1].strip
+      end
+
+      ## try to find country
+      cty = Country.search_by_name( country_name )
+
+      if cty.nil?
+        logger.warn "no country match found for >#{country_name}<; skipping line; in [#{name}]"
+        next
+      end
+
+      if name =~ /\/fifa/
+        cty.fifa = values[0].strip
+      elsif name =~ /\/fips/
+        cty.fips = values[0].strip
+      elsif name =~ /\/internet/
+        # NOTE: remove (optional) leading . e.g. .at becomes at
+        cty.net  = values[0].sub( /^\s*\./,'' ).strip  
+      elsif name =~ /\/ioc/
+        cty.ioc = values[0].strip
+      elsif name =~ /\/motor/
+        cty.motor = values[0].strip
+      elsif name =~ /\/iso/
+        cty.alpha2 = values[1].strip
+        cty.alpha3 = values[2].strip
+        # NOTE: num is a string!!! use (rename to) num_str - why? why not?
+        cty.num    = values[3].strip
+      else
+        logger.warn "warn: unknown country code type; skipping line; in [#{name}]"
+        next
+      end
+
+      cty.save!
+    end
+  end # method load_codes
+
+
+####
+# helper methods
+## todo: also add City.search_by_name etc. !!!
+
+
+end # class ReaderBase
+end # module WorldDb

data/lib/worlddb/reader_file.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class Reader < ReaderBase
+
+  def initialize( include_path, opts={} )
+    super( opts )
+
+    @include_path = include_path
+  end
+
+
+  def create_fixture_reader( name )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (setup) '#{name}' (#{path})..."
+
+    FixtureReader.from_file( path )
+  end
+
+  def create_lang_reader( name )
+    path = "#{@include_path}/#{name}.yml"   ## hash reader - use .yml??
+    logger.info "parsing data (lang) '#{name}' (#{path})..."
+
+    LangReader.from_file( path )
+  end
+
+  def create_usage_reader( name )
+    path = "#{@include_path}/#{name}.yml"   ## hash reader - use .yml??
+    logger.info "parsing data (usage) '#{name}' (#{path})..."
+
+    UsageReader.from_file( path )
+  end
+
+
+  def create_country_reader( name, more_attribs={} )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (country) '#{name}' (#{path})..."
+
+    CountryReader.from_file( path, more_attribs )
+  end
+
+  def create_region_reader( name, more_attribs={} )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (region) '#{name}' (#{path})..."
+
+    RegionReader.from_file( path, more_attribs )
+  end
+
+  def create_city_reader( name, more_attribs={} )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (city) '#{name}' (#{path})..."
+
+    CityReader.from_file( path, more_attribs )
+  end
+
+
+  def create_hash_reader( name )
+    path = "#{@include_path}/#{name}.yml"
+    logger.info "parsing data (hash) '#{name}' (#{path})..."
+
+    HashReader.from_file( path )
+  end
+
+  def create_values_reader( name, more_attribs={} )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (values) '#{name}' (#{path})..."
+
+    ValuesReader.from_file( path, more_attribs )
+  end
+
+  def create_line_reader( name )
+    path = "#{@include_path}/#{name}.txt"
+    logger.info "parsing data (line) '#{name}' (#{path})..."
+
+    LineReader.from_file( path )
+  end
+
+#  def create_tag_reader( name )
+#    ## fix: change to new from_file() style
+#    TagDb::TagReader.new( @include_path )
+#  end
+
+
+end # class Reader
+end # module WorldDb

data/lib/worlddb/reader_zip.rb

@@ -0,0 +1,160 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class ZipReader < ReaderBase
+
+
+  def initialize( name, include_path, opts = {} )
+    super( opts )
+
+    ## todo/fix: make include_path an opts (included in opts?) - why? why not??
+    path = "#{include_path}/#{name}.zip"
+    
+    ## todo: check if zip exists
+    @zip_file = Zip::File.open( path ) ## NOTE: do NOT create if file is missing; let it crash
+
+    ### allow prefix (path) in name
+    ### e.g. assume all files relative to setup manifest
+    ## e.g. at-austria-master/setups/all.txt or
+    ## be-belgium-master/setups/all.txt
+    ## for
+    ## setups/all.txt
+    ###
+    ## will get (re)set w/ fixture/setup reader
+    ##
+    ## todo/fix: change/rename to @relative_path ?? - why? why not?
+    @zip_prefix = ''
+  end
+
+  def close
+    ## todo/check: add a close method - why? why not ???
+    @zip_file.close
+  end
+
+
+
+  def create_fixture_reader( name )
+    ## e.g. pass in => setups/all or setups/test etc. e.g. w/o .txt extension
+    query = "**/#{name}.txt"
+
+    ## note: returns an array of Zip::Entry
+    candidates = @zip_file.glob( query )
+    pp candidates
+
+    ## use first candidates entry as match
+    ## todo/fix: issue warning if more than one entries/matches!!
+
+    ## get fullpath e.g. at-austria-master/setups/all.txt
+    path = candidates[0].name
+    logger.debug "  zip entry path >>#{path}<<"
+
+    ## cut-off at-austria-master/    NOTE: includes trailing slash (if present)
+    ## logger.debug "  path.size #{path.size} >>#{path}<<"
+    ## logger.debug "  name.size #{name.size+4} >>#{name}<<"
+
+    ## note: add +4 for extension (.txt)
+    @zip_prefix = path[ 0...(path.size-(name.size+4)) ]
+    logger.debug "  zip entry prefix >>#{@zip_prefix}<<"
+
+    logger.info "parsing data (setup) in zip '#{name}' (#{path})..."
+
+    FixtureReader.from_zip( @zip_file, path )
+  end
+
+
+  def create_lang_reader( name )
+    path = name_to_zip_entry_path( name, '.yml' )  ## hash reader - use .yml??
+    logger.info "parsing data (lang) in zip '#{name}' (#{path})..."
+
+    LangReader.from_zip( @zip_file, path )
+  end
+
+  def create_usage_reader( name )
+    path = name_to_zip_entry_path( name, '.yml' )  ## hash reader - use .yml??
+    logger.info "parsing data (usage) in zip '#{name}' (#{path})..."
+
+    UsageReader.from_zip( @zip_file, path )
+  end
+
+
+  def create_country_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (country) in zip '#{name}' (#{path})..."
+
+    CountryReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_region_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (region) in zip '#{name}' (#{path})..."
+
+    RegionReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_city_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (city) in zip '#{name}' (#{path})..."
+
+    CityReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+
+  def create_hash_reader( name )
+    path = name_to_zip_entry_path( name, '.yml' )  ## hash reader - use .yml??
+    logger.info "parsing data (hash) in zip '#{name}' (#{path})..."
+
+    HashReader.from_zip( @zip_file, path )
+  end
+
+  def create_values_reader( name, more_attribs={} )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (values) in zip '#{name}' (#{path})..."
+
+    ValuesReader.from_zip( @zip_file, path, more_attribs )
+  end
+
+  def create_line_reader( name )
+    path = name_to_zip_entry_path( name )
+    logger.info "parsing data (line) in zip '#{name}' (#{path})..."
+
+    LineReader.from_zip( @zip_file, path )
+  end
+
+#  def create_tag_reader( name )
+#    ## fix: change to new from_file() style
+#    TagDb::TagReader.new( @include_path )
+#  end
+
+private
+
+  def path_to_real_path( path )
+    # map name to name_real_path
+    # name might include !/ for virtual path (gets cut off)
+    # e.g. at-austria!/w-wien/beers becomse w-wien/beers
+    pos = path.index( '!/')
+    if pos.nil?
+      path # not found; real path is the same as name
+    else
+      # cut off everything until !/ e.g.
+      # at-austria!/w-wien/beers becomes
+      # w-wien/beers
+      path[ (pos+2)..-1 ]
+    end
+  end
+
+  def name_to_zip_entry_path( name, extension='.txt' )
+    path = "#{name}#{extension}"
+
+    real_path = path_to_real_path( path )
+
+    # NOTE: add possible zip entry prefix path
+    #          (if present includes trailing slash e.g. /)
+    entry_path = "#{@zip_prefix}#{real_path}"
+    entry_path
+  end
+
+
+
+end # class ZipReader
+end # module WorldDb

data/lib/worlddb/readers/city.rb

@@ -0,0 +1,81 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class CityReader
+
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  def self.from_zip( zip_file, entry_path, more_attribs={} )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text, more_attribs )
+  end
+
+
+  def self.from_file( path, more_attribs={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_string( text, more_attribs={} )
+    CityReader.new( text, more_attribs )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, more_attribs={} )
+    ## todo/fix: how to add opts={} ???
+
+    @text         = text
+    @more_attribs = more_attribs
+  end
+
+
+  def read()
+    reader = ValuesReader.from_string( @text, @more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      City.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class CityReader
+end # module WorldDb
+

data/lib/worlddb/readers/country.rb

@@ -0,0 +1,78 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class CountryReader
+
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  def self.from_zip( zip_file, entry_path, more_attribs={} )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_file( path, more_attribs={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, more_attribs )    
+  end
+
+  def self.from_string( text, more_attribs={} )
+    CountryReader.new( text, more_attribs )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, more_attribs={} )
+    ## todo/fix: how to add opts={} ???
+
+    @text         = text
+    @more_attribs = more_attribs
+  end
+
+  def read()
+    reader = ValuesReader.from_string( @text, @more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      Country.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class CountryReader
+end # module WorldDb