factbook 2.0.0 → 2.0.1

data/lib/factbook/builder_json.rb

@@ -1,79 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-######
-# json builder -- lets us rebuild a page from "dumped" json (instead of parsing html page)
-
-class JsonBuilder    
-  include LogUtils::Logging
-  include NormalizeHelper    ##  e.g. normalize_category
-
-
-def self.from_file( path )
-  text = File.read( path )     ## fix: use File.read_utf8  from textutils
-  self.from_string( text )
-end
-
-def self.from_string( text )
-  self.new( text )
-end
-
-
-attr_reader :text,
-            :json,
-            :info,            ## not used yet -- page info incl. country_name, region_name, last_updated etc.
-            :errors,          ## not used yet -- encoding erros etc.
-            :sects
-
-
-def initialize( text )
-  @text = text
-    
-  @json = JSON.parse( text )
-
-  @info   = nil   ## fix/todo: sorry - for now no page info (use header in json - why? why not??)
-  @errors = []       ## fix/todo: sorry - for now no errors possible/tracked
-
-  @sects = []
-
-  @json.each do |k1,v1|
-    sect_title    = k1
-    sect_subsects = v1
-    
-    sect = Sect.new
-    sect.title = sect_title
-    
-    ## get subsections
-    subsects = []
-    sect_subsects.each do |k2,v2|
-      subsect_title = k2
-      subsect_data  = v2
-      
-      subsect = Subsect.new
-      subsect.title = subsect_title
-
-      #####
-      ## note: run data hash through normalize_category (again)
-      if subsect_data.is_a?( Hash )
-        new_subsect_data = {}
-        subsect_data.each do |k3,v3|
-          new_subsect_data[ normalize_category(k3) ] = v3
-        end
-        subsect_data = new_subsect_data
-      end  
- 
-      subsect.data  = subsect_data
-      
-      subsects << subsect
-    end
-    
-    sect.subsects = subsects
-    @sects << sect
-  end
-end
-
-end # class JsonBuilder
-
-
-end # module Factbook

data/lib/factbook/codes.rb

@@ -1,119 +0,0 @@
-# encoding: utf-8
-
-##
-# note:
-#   the factbook category/region for world is other entities (on FAQ) and oceans in page
-#    changed to world
-
-
-module Factbook
-
-class Codes
-
-  Code = Struct.new( :code,      ## todo: add notes (country affiliation) - why? why not??
-                     :name,
-                     :category,  ## e.g. Countries, Other, Oceans, World, Dependencies, etc.
-                     :region,    ## e.g. Europe, Oceans, etc.
-                    )
-
-  def self.from_csv( path )
-    ###
-    #  note:
-    #   if you use quotes - NO leading spaces allowed e.g.
-    #  use au,"Austria",... and NOT
-    #      au, "Austria", ...
-    #
-    #  for headers - NO leading spaces allowed e.g.
-    #   use  Code,Name,Category,Region,...   and NOT
-    #        Code, Name, Category, Region, ...
-
-    rows = CSV.read( path, headers: true )
-    
-    pp rows
-
-    recs = []
-    rows.each do |row|
-      pp row
-      rec = Code.new
-      rec.code     = row['Code'].strip    ## remove leading n trailing whitespaces
-      rec.name     = row['Name'].strip
-
-      ## note: for now category and region are optional
-      rec.category = row['Category'].strip    if row['Category']
-      rec.region   = row['Region'].strip      if row['Region']  
-
-      pp rec
-      recs << rec
-    end
-
-    self.new( recs )
-  end
-
-  def initialize( codes )
-    @codes = codes
-  end
-  
-  def size() @codes.size; end
-
-  def each
-    @codes.each {|code| yield( code ) }
-  end
-
-  def to_a
-    @codes.collect {|code| code.code }   ## return array of codes
-  end
-
-  ##  def all()  self.to_a; end    ## note: alias for to_a - use - why? why not??
-
-  ## "pre-defined" convenience shortcuts
-  def countries()       category 'Countries';     end
-  def world()           category 'World';         end
-  def oceans()          category 'Oceans';        end
-  def misc()            category 'Miscellaneous'; end
-  def others()          category 'Other';         end
-  def dependencies()    category 'Dependencies';  end
-  def dependencies_us() category 'Dependencies (United States)'; end
-## fix/todo: add all dependencies  uk (or gb?), fr,cn,au,nz,no,dk,etc.
-
-  def europe()               region 'Europe';            end
-  def south_asia()           region 'South Asia';        end
-  def central_asia()         region 'Central Asia';      end
-  def east_n_souteast_asia() region 'East & Southeast Asia'; end
-  def middle_east()          region 'Middle East';       end
-  def africa()               region 'Africa';            end
-  def north_america()        region 'North America';     end
-  def central_america_n_caribbean() region 'Central America and Caribbean'; end
-  def south_america()        region 'South America';     end
-  def australia_oceania()    region 'Australia-Oceania'; end
-  def antartica()            region 'Antarctica';        end
-
-  ## note: regions oceans and world - same as category oceans and world
-  ##     use oceans_ii or world_ii or something ??
-  ##   use category('World')  n region('World')
-  ##   use category('Oceans') n region('Oceans')
-
-
-  def category( query )
-    ## todo/future: allow passing in of regex too (not just string)
-    ## note: e.g. Dependencies (France) needs to get escpaed to
-    ##            Dependencies \(France\)  etc.
-    filter_regex = /#{Regexp.escape(query)}/i
-    codes = @codes.select do |code|
-      code.category ? filter_regex.match( code.category ) : false   ## note: allow nil for category; will fail on search
-    end
-    Codes.new( codes )   ## return new Codes obj for easy-chaining
-  end
-
-  def region( query )
-    ## todo/future: allow passing in of regex too (not just string)
-    filter_regex = /#{Regexp.escape(query)}/i
-    codes = @codes.select do |code|
-       code.region ? filter_regex.match( code.region ) : false      ## note: allow nil for region; will fail on search
-    end
-    Codes.new( codes )   ## return new Codes obj for easy-chaining
-  end
-
-end  # class codes
-
-end # module Factbook
-

data/lib/factbook/comparisons.rb

@@ -1,50 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-class Comparisons
-
-  Comparison = Struct.new( :num,        ### todo: use no or id or something - why? why not?
-                           :category,  ## e.g. Geography, People, Economy, etc.
-                           :name,
-                          )
-
-  def self.from_csv( path )
-
-    rows = CSV.read( path, headers: true )
-    
-    pp rows
-
-    recs = []
-    rows.each do |row|
-      pp row
-      rec = Comparison.new
-      rec.num      = row['Num'].strip.to_i    ## remove leading n trailing whitespaces
-      rec.category = row['Category'].strip
-      rec.name     = row['Name'].strip
-
-      pp rec
-      recs << rec
-    end
-
-    self.new( recs )
-  end
-
-  def initialize( comps )
-    @comps = comps
-  end
-  
-  def size() @comps.size; end
-
-  def each
-    @comps.each {|comp| yield( comp ) }
-  end
-
-  def to_a
-    @comps.collect {|comp| comp.num }   ## return array of nums   -- return something else - why? why not?
-  end
-
-end  # class Comparison
-
-end # module Factbook
-

data/lib/factbook/counter.rb

@@ -1,48 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-class Counter
-
-attr_reader :data
-
-def initialize
-  @data = {}
-end
-
-def count( page )
-
-  ## walk page data hash
-  #   add nodes to data
-
-  walk( page, page.data, @data )
-end
-
-
-private
-def walk( page, hin, hout )
-   hin.each do |k,v|
-     if v.is_a? Hash
-        hout2 =  hout[k] || { count: 0, codes: '' }
-        
-        hout2[ :count ] += 1
-        
-        ## delete codes if larger (treshhold) than x (e.g. 9)
-        hout2.delete( :codes )    if hout2[ :count ] > 9
-
-        codes = hout2[ :codes ]
-        if codes    ## note: might got deleted if passed treshhold (e.g. 9 entries)
-          codes << ' '  unless codes.empty?   ## add separator (space for now)
-          codes << page.info.country_code 
-          hout2[ :codes ] = codes
-        end
-             
-        hout[k] = hout2        
-        walk( page, v, hout2 )
-     end
-   end
-end
-
-end # class Counter
-
-end # module Factbook

data/lib/factbook/db/importer.rb

@@ -1,92 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-class Importer
-
-def import( page )
-
-  ## note: assumes active connection
-
-  code  = page.info.country_code
-  name  = page.info.country_name
-
-  attribs = {
-   name:       name,
-   area:       sq_km( page.area ),        # e.g. 83,871 sq km       
-   area_land:  sq_km( page.area_land ),   # e.g. 82,445 sq km
-   area_water: sq_km( page.area_water ),  # e.g.  1,426 sq km
-
-   population:         num( page.population ),                   # e.g.  8,665,550 (July 2015 est.)  
-   population_growth:  percent( page.population_growth ),        # e.g.  0.55% (2015 est.)
-   birth_rate:         rate_per_thousand( page.birth_rate ),     # e.g.  9.41 births/1,000 population (2015 est.)
-   death_rate:         rate_per_thousand( page.death_rate ),     # e.g.  9.42 deaths/1,000 population (2015 est.)
-   migration_rate:     rate_per_thousand( page.migration_rate ), # e.g.  5.56 migrant(s)/1,000 population (2015 est.)
-  }
-
-  rec = Fact.find_by( code: code )
-  if rec.nil?   ## create (new) record
-    rec = Fact.new
-    attribs[ :code ] = code
-    puts "create fact record #{code}/#{name}:"
-  else          ## update (exisiting) record
-    puts "update fact record #{code}/#{name}:"
-  end
-  
-  puts "  #{attribs.inspect}"
-  rec.update_attributes!( attribs )
-end
-
-
-def rate_per_thousand( text )
-  # e.g.  9.41 births/1,000 population (2015 est.)
-  #       9.42 deaths/1,000 population (2015 est.)
-  #       5.56 migrant(s)/1,000 population (2015 est.)
-
-  if text =~/([0-9\.]+) [a-z\(\)]+\/1,000/
-    $1.to_f
-  else
-    puts "*** warn: unknown rate <name>/1,000 format (no match): >#{text}<"    
-    nil
-  end
-end
-
-def num( text )
-  # e.g. 8,665,550 (July 2015 est.) 
-
-  if text =~/([0-9,\.]+)/
-    $1.gsub(',', '').to_i   ## note: remove commas (,) if present
-  else
-    puts "*** warn: unknown number format (no match): >#{text}<"
-    nil   ## return nil
-  end
-end
-
-def percent( text )
-  # e.g. 0.55% (2015 est.)
-
-  if text =~/([0-9\.]+)%/
-    $1.to_f
-  else
-    puts "*** warn: unknown percent format (no match): >#{text}<"
-    nil   ## return nil
-  end
-end
-
-def sq_km( text )
-  # e.g. 83,871 sq km
-  ##   todo - check vatican - uses float e.g. 0.44 ?? add support?
-  
-  if text =~/([0-9,\.]+) sq km/
-    $1.gsub(',', '').to_i   ## note: remove commas (,) if present
-  else
-    puts "*** warn: unknown sq km format (no match): >#{text}<"
-    nil   ## return nil
-  end
-end
-
-
-end # class Importer
-
-end # module Factbook
-

data/lib/factbook/db/models.rb

@@ -1,11 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-
-class Fact < ActiveRecord::Base
-    
-end  # class Fact
-
-
-end # module Factbook

data/lib/factbook/db/schema.rb

@@ -1,36 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-class CreateDb
-
-def up
-
-ActiveRecord::Schema.define do
-
-create_table :facts do |t|
-  t.string  :code,   null: false   # country code e.g. au
-  t.string  :name,   null: false   # country name e.g. Austria
-
-  t.integer :area        # e.g. 83,871 sq km       
-  t.integer :area_land   # e.g. 82,445 sq km    --use float - why? why not?
-  t.integer :area_water  # e.g.  1,426 sq km
-
-  t.integer :population            # e.g.  8,665,550 (July 2015 est.)  
-  t.float   :population_growth     # e.g.  0.55% (2015 est.)
-  t.float   :birth_rate            # e.g.  9.41 births/1,000 population (2015 est.)
-  t.float   :death_rate            # e.g.  9.42 deaths/1,000 population (2015 est.)
-  t.float   :migration_rate        # e.g.  5.56 migrant(s)/1,000 population (2015 est.)
-
-  t.timestamps
-end
-
-
-end # block Schema.define
-
-end # method up
-
-
-end # class CreateDb
-
-end # module Factbook

data/lib/factbook/normalize.rb

@@ -1,43 +0,0 @@
-# encoding: utf-8
-
-module Factbook  
-  module NormalizeHelper
-
-
-def normalize_category( text )
-
-  ## note: fix typos/errors with double colons e.g. note::  (instead of note:)
-
-  text = text.strip
-  text = text.sub( /:+\z/, '' )      # remove trailing : if present -- note: allow (fix) note:: too, thus, use :+
-  text = text.strip
-
-  #######################################
-  ### special cases
-
-  ##   typos e.g ntoe => use note
-  text = 'note'                         if text == 'ntoe'
-  text = 'investment in fixed capital'  if text == 'investment if fixed capital'  
-
-  ##  downcase
-  text = 'lowest point'    if text == 'Lowest point'
-  text = 'chief of state'  if text == 'Chief of state'
-
-  ##  spelling variant (use more popular one)
-  text = 'signed, but not ratified'     if text == 'signed but not ratified'
-  text = 'vectorborne diseases'         if text == 'vectorborne disease'
-  text = 'water contact disease'        if text == 'water contact diseases'
-  text = 'food or waterborne diseases'  if text == 'food or waterborne disease'
-  text = 'geographic coordinates'       if text == 'geographical coordinates'
-  text = 'note'                         if text == 'notes'
-  text = 'refugees (country of origin)' if text == 'refugees (countries of origin)'
-
-  ##    border countries (8):   -- remove (x) counter
-  text = 'border countries'   if text.start_with?( 'border countries')
-    
-  text
-end
-
-
-  end # module NormalizeHelper
-end  # module Factbook