worlddb-models 2.2.2 → 2.3.0

data/lib/worlddb/readers/base.rb

@@ -0,0 +1,149 @@
+# encoding: UTF-8
+
+module WorldDb
+
+
+###
+# todo/fix:
+#   try to merge ReaderBaseWithMoreAttribs and ReaderBaseWithOpts into one base - why? why not?
+
+
+class ReaderBaseWithMoreAttribs
+
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  def self.from_zip( zip_file, entry_path, more_attribs={} )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_file( path, more_attribs={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, more_attribs )
+  end
+
+  def self.from_string( text, more_attribs={} )
+    puts "[debug] ReaderBase.from_string calling #{self.name}.new"  # note: assume self is derived class (object)
+    self.new( text, more_attribs )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, more_attribs={} )
+    ## todo/fix: how to add opts={} ???
+
+    @text         = text
+    @more_attribs = more_attribs
+  end
+
+end # class ReaderBaseWithMoreAttribs
+
+
+
+class ReaderBaseWithOpts
+
+  include LogUtils::Logging
+
+## make models available by default with namespace
+#  e.g. lets you use Usage instead of Model::Usage
+  include Models
+
+## value helpers e.g. is_year?, is_taglist? etc.
+  include TextUtils::ValueHelper
+
+
+  ## todo: add opts={} etc.
+  def self.from_zip( zip_file, entry_path )
+    ## get text content from zip
+
+    entry = zip_file.find_entry( entry_path )
+
+    ## todo/fix: add force encoding to utf-8 ??
+    ##  check!!!
+    ##  clean/prepprocess lines
+    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
+    text = entry.get_input_stream().read()
+
+    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
+    logger = LogUtils::Logger.root
+    logger.debug "text.encoding.name (before): #{text.encoding.name}"
+#####
+# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+## NB:
+# for now "hardcoded" to utf8 - what else can we do?
+# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    text = text.force_encoding( Encoding::UTF_8 )
+    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
+
+    ## todo:
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
+
+    self.from_string( text )
+  end
+
+  def self.from_file( path, opts={} )
+    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+    text = File.read_utf8( path )
+    self.from_string( text, opts )
+  end
+
+  def self.from_string( text, opts={} )
+    puts "[debug] ReaderBase.from_string calling #{self.name}.new"  # note: assume self is derived class (object)
+    self.new( text, opts )
+  end
+
+
+  def skip_tags?()   @skip_tags == true;  end
+  def strict?()      @strict == true;     end
+
+  def initialize( text, opts={} )
+    @text = text
+
+    ## option: do NOT generate/add any tags for countries/regions/cities
+    @skip_tags =  opts[:skip_tags].present? ? true : false
+    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
+    @strict    =  opts[:strict].present? ? true : false
+  end
+
+end # class ReaderBaseWithOpts
+
+
+end # module WorldDb

data/lib/worlddb/readers/city.rb

@@ -2,72 +2,9 @@
 
 module WorldDb
 
-class CityReader
+class CityReader < ReaderBaseWithMoreAttribs
 
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-
-  def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text, more_attribs )
-  end
-
-
-  def self.from_file( path, more_attribs={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, more_attribs )
-  end
-
-  def self.from_string( text, more_attribs={} )
-    CityReader.new( text, more_attribs )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, more_attribs={} )
-    ## todo/fix: how to add opts={} ???
-
-    @text         = text
-    @more_attribs = more_attribs
-  end
-
-
-  def read()
+  def read
     reader = ValuesReader.from_string( @text, @more_attribs )
 
     reader.each_line do |attribs, values|

data/lib/worlddb/readers/country.rb

@@ -2,70 +2,9 @@
 
 module WorldDb
 
-class CountryReader
+class CountryReader < ReaderBaseWithMoreAttribs
 
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-
-  def self.from_zip( zip_file, entry_path, more_attribs={} )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text, more_attribs )
-  end
-
-  def self.from_file( path, more_attribs={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, more_attribs )    
-  end
-
-  def self.from_string( text, more_attribs={} )
-    CountryReader.new( text, more_attribs )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, more_attribs={} )
-    ## todo/fix: how to add opts={} ???
-
-    @text         = text
-    @more_attribs = more_attribs
-  end
-
-  def read()
+  def read
     reader = ValuesReader.from_string( @text, @more_attribs )
 
     reader.each_line do |attribs, values|

data/lib/worlddb/readers/lang.rb

@@ -2,73 +2,9 @@
 
 module WorldDb
 
-class LangReader
+class LangReader < ReaderBaseWithOpts
 
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-
-  ## todo: add opts={} etc.
-  def self.from_zip( zip_file, entry_path )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text )
-  end
-
-  def self.from_file( path, opts={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, opts )
-  end
-
-  def self.from_string( text, opts={} )
-    LangReader.new( text, opts )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, opts={} )
-    @text = text
-
-    ## option: do NOT generate/add any tags for countries/regions/cities
-    @skip_tags =  opts[:skip_tags].present? ? true : false
-    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
-    @strict    =  opts[:strict].present? ? true : false
-  end
-
-  def read()
+  def read
     reader = HashReader.from_string( @text )
 
     reader.each do |key, value|
@@ -100,8 +36,7 @@ class LangReader
         lang.update_attributes!( lang_attribs )
     end # each key,value
 
-  end # method load_langs
-
+  end # method read
 
 end # class LangReader
 end # module WorldDb

data/lib/worlddb/readers/state.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class StateReader < ReaderBaseWithMoreAttribs
+
+  def read
+    reader = ValuesReader.from_string( @text, @more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      State.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class StateReader
+
+
+class PartReader < ReaderBaseWithMoreAttribs
+
+  def read
+    reader = ValuesReader.from_string( @text, @more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      Part.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class PartReader
+
+
+class CountyReader < ReaderBaseWithMoreAttribs
+
+  def read
+    reader = ValuesReader.from_string( @text, @more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      County.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class CountyReader
+
+
+class MuniReader < ReaderBaseWithMoreAttribs
+
+  def read
+    reader = ValuesReader.from_string( @text, @more_attribs )
+
+    reader.each_line do |attribs, values|
+      opts = { skip_tags: skip_tags? }
+      Muni.create_or_update_from_attribs( attribs, values, opts )
+    end
+  end
+
+end # class MuniReader
+
+
+end # module WorldDb

data/lib/worlddb/readers/state_tree.rb

@@ -0,0 +1,118 @@
+# encoding: UTF-8
+
+module WorldDb
+
+class StateTreeReader < ReaderBaseWithMoreAttribs
+
+  def read
+    ## for now requires country_id !!
+    country = Country.find( @more_attribs[ :country_id ] )
+    puts "[StateTreeReader] country key: #{country.key}, name: #{country.name}"
+
+    reader = TreeReader.from_string( @text )
+
+    stack = []    # note: last_level  => stack.size; starts w/ 0
+
+    if country.key == 'de'
+      ## use state (1) > part (2) > county (3) > muni (4) > city (5)
+      state_level  = 1   # Land
+      part_level   = 2   # Regierungsbezirk
+      county_level = 3   # Landkreis, Kreisfreie Stadt
+      muni_level   = 4   # Gemeinde
+      city_level   = 5   # Stadt, Ort, etc.
+    else
+      ## use state (1) > county (2) > muni (3) > city (4)
+      state_level  = 1
+      part_level   = -1    ## note: not in use (-1)
+      county_level = 2
+      muni_level   = 3
+      city_level   = 4
+    end
+
+    reader.each_line do |nodes|
+      names = nodes.map { |item| "(#{item.level}) #{item.value}" }
+      node = nodes.last
+
+      puts "    #{names.join( ' › ' )}:"
+      puts "      key: >#{node.key}<, level: >#{node.level}<, value: >#{node.value}<"
+
+      if node.level == state_level  # 1
+        rec = State.where( "name like '#{node.value}%'" ).first 
+      elsif node.level == part_level # 2 
+        state = stack[0]
+        rec = Part.where( "name like '#{node.value}%' AND state_id = #{state.id}" ).first
+      elsif node.level == county_level  # 2 or 3
+        state = stack[0]
+        rec = County.where( "name like '#{node.value}%' AND state_id = #{state.id}" ).first
+      elsif node.level == muni_level   # 3 or 4
+        state = stack[0]
+        rec = Muni.where( "name like '#{node.value}%' AND state_id = #{state.id}" ).first
+      elsif node.level == city_level  # 4 or 5
+        ## note: city requires country scope for lookup
+        ## todo/fix: how to deal with cities with the same name
+        ##   in the same country (and same state and same county etc.) ??? - add some examples here
+        rec = City.where( "name like '#{node.value}%' AND country_id = #{country.id}" ).first
+      else
+        puts "*** (fatal) error: unknown level for tree node: #{node.inspect}"
+        ## todo/fix: exit here
+      end
+
+
+      if rec.present?
+        puts "ok - record match found: #{rec.inspect}"
+      else
+        ## note: for now only auto-adds munis n cities
+        if node.level == muni_level  # 3 or 4
+          ## add muni
+          key    = TextUtils.title_to_key( node.value )
+          name   = node.value
+          level  = node.level
+          state  = stack[0]
+          county = stack[county_level-1]  # note: stack is zero-based (thus, -1)
+          puts "*** adding muni record:"
+          rec = Muni.create!( key:       key,
+                              name:      name,
+                              level:     level,
+                              state_id:  state.id,
+                              county_id: county.id )
+        elsif node.level == city_level # 4 or 5
+          ## add city
+          key    = TextUtils.title_to_key( node.value )
+          name   = node.value
+          state  = stack[0]
+          county = stack[county_level-1]  # note: stack is zero-based (thus, -1)
+          muni   = stack[muni_level-1]    # note: stack is zero-based (thus, -1)
+          puts "*** adding city record:"
+          rec = City.create!( key:        key,
+                              name:       name,
+                              state_id:   state.id,
+                              ## add county_id too ???
+                              muni_id:    muni.id,
+                              country_id: country.id )
+        else
+          puts "*** (fatal) error: record not found for tree node: #{node.inspect}"
+          ## todo/fix: exit here
+        end
+      end
+
+      level_diff = node.level - stack.size
+
+      if level_diff > 0
+        logger.debug "[StateTreeReader]    up  +#{level_diff}"
+        ## FIX!!! todo/check/verify/assert: always must be +1
+      elsif level_diff < 0
+        logger.debug "[StateTreeReader]    down #{level_diff}"
+        level_diff.abs.times { stack.pop }
+        stack.pop
+      else
+        ## same level
+        stack.pop
+      end
+      stack.push( rec )   ## hierarchy of records  (mirrors hierarchy of read-in text)
+
+    end # each_line
+  end # method read
+
+end   # class StateTreeReader
+end   # module WorldDb
+

data/lib/worlddb/readers/usage.rb

@@ -2,72 +2,9 @@
 
 module WorldDb
 
-class UsageReader
+class UsageReader < ReaderBaseWithOpts
 
-  include LogUtils::Logging
-
-## make models available by default with namespace
-#  e.g. lets you use Usage instead of Model::Usage
-  include Models
-
-## value helpers e.g. is_year?, is_taglist? etc.
-  include TextUtils::ValueHelper
-
-  ## todo: add opts
-  def self.from_zip( zip_file, entry_path )
-    ## get text content from zip
-
-    entry = zip_file.find_entry( entry_path )
-
-    ## todo/fix: add force encoding to utf-8 ??
-    ##  check!!!
-    ##  clean/prepprocess lines
-    ##  e.g. CR/LF (/r/n) to LF (e.g. /n)
-    text = entry.get_input_stream().read()
-
-    ## NOTE: needs logger ref; only available in instance methods; use global logger for now
-    logger = LogUtils::Logger.root
-    logger.debug "text.encoding.name (before): #{text.encoding.name}"
-#####
-# NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-## NB:
-# for now "hardcoded" to utf8 - what else can we do?
-# - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    text = text.force_encoding( Encoding::UTF_8 )
-    logger.debug "text.encoding.name (after): #{text.encoding.name}"     
-
-    ## todo:
-    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
-    ## text = TextUtils.convert_unicode_dashes_to_plain_ascii( text, path: path )
-
-    self.from_string( text )
-  end
-
-  def self.from_file( path, opts={} )
-    ## note: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see textutils/utils.rb
-    text = File.read_utf8( path )
-    self.from_string( text, opts )
-  end
-
-  def self.from_string( text, opts={} )
-    UsageReader.new( text, opts )
-  end
-
-
-  def skip_tags?()   @skip_tags == true;  end
-  def strict?()      @strict == true;     end
-
-  def initialize( text, opts={} )
-    @text = text
-
-    ## option: do NOT generate/add any tags for countries/regions/cities
-    @skip_tags =  opts[:skip_tags].present? ? true : false
-    ## option: for now issue warning on update, that is, if key/record (country,region,city) already exists
-    @strict    =  opts[:strict].present? ? true : false
-  end
-
-  def read()
+  def read
     reader = HashReader.from_string( @text )
 
     reader.each do |key, value|