factbook 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9715102d9e477fecc63d0b77e0760c134c349edf
-  data.tar.gz: 26f7ec5f6f517013e96f477682d92b61bfcd3103
+  metadata.gz: 24d96765bcda05d51ce7f72baa5f565da45aa85b
+  data.tar.gz: 7841f960d9905ad051c6ef255f374ed875091f88
 SHA512:
-  metadata.gz: 843926700c08da16bb1a7555ed321d187e08cc8403c2855e2c240668462b24d2127622f5cf8d980ca23804cca9a8ef982e910554fab90559d6c776ff97b42217
-  data.tar.gz: 31cf12c0d3e063b84f5be8794b1dcc9ad33ef6bc075bd478a10781c5408feef1766fc83ec6391603eee9eade7ea7d1e1d098ca27c63c90a7364866165d171118
+  metadata.gz: 93fac1f61de8fcd72e98afb4ed5bb6619cb849fac69ef979c2a5703fd2886a0cf8b6a9c260cb179c3209d6d61a57170ac7dc11e81c59928ec3532efcbf212180
+  data.tar.gz: 5e84df0a02a7278b9cb54f19fd02364abe302c78a1aa222b334e3844141b13d82c10e1e51b53d5a906bbb8f43ee59cd44b02abde2aff58c6f1cff37355346ce8

data/Manifest.txt

@@ -11,6 +11,9 @@ lib/factbook/builder.rb
 lib/factbook/builder_item.rb
 lib/factbook/codes.rb
 lib/factbook/comparisons.rb
+lib/factbook/db/importer.rb
+lib/factbook/db/models.rb
+lib/factbook/db/schema.rb
 lib/factbook/page.rb
 lib/factbook/sanitizer.rb
 lib/factbook/sect.rb
@@ -28,10 +31,13 @@ test/data/be.yml
 test/data/src/au.html
 test/data/src/be.html
 test/helper.rb
+test/test_attribs.rb
 test/test_builder.rb
 test/test_codes.rb
 test/test_comparisons.rb
+test/test_convert.rb
 test/test_fields.rb
+test/test_importer.rb
 test/test_item_builder.rb
 test/test_json.rb
 test/test_page.rb

data/README.md

@@ -64,6 +64,33 @@ resulting in:
     ...
 ```
 
+### Use shortcut attribute accessors
+
+```ruby
+pp page.background        ## same as page['Introduction']['Background']['text']
+# => "Following more than three centuries..."
+pp page.area              ## same as page['Geography'][''Area']['total']['text']
+# => "8,515,770 sq km"
+pp page.area_land         ## same as page['Geography'][''Area']['land']['text']
+# => "8,358,140 sq km"
+pp page.area_water        ## same as page['Geography'][''Area']['water']['text']
+# => "157,630 sq km"
+pp page.area_note         ## same as page['Geography'][''Area']['note']['text']
+# => "includes Arquipelago de Fernando de Noronha, Atol das Rocas, ..."
+pp page.area_comparative  ## same as page['Geography']['Area - comparative']['text']
+# => "slightly smaller than the US" 
+pp page.climate           ## same as page['Geography']['Climate']['text']
+# => "mostly tropical, but temperate in south"
+pp page.terrain           ## same as page['Geography']['Terrain']['text']
+# => "mostly flat to rolling lowlands in north; ..."
+pp page.elevation_lowest  ## same as page['Geography']['Elevation extremes']['lowest point']['text']
+# => "Atlantic Ocean 0 m"
+pp page.elevation_highest ## same as page['Geography']['Elevation extremes']['highest point']['text']
+# => "Pico da Neblina 2,994 m"
+pp page.resources         ## same as page['Geography'][Natural resources']['text']
+# => "bauxite, gold, iron ore, manganese, nickel, phosphates, ..."
+...
+```
 
 ### Save to disk as JSON
 
@@ -75,7 +102,7 @@ end
 ```
 
 
-### Print all codes
+### List all codes
 
 ```ruby
 Factbook.codes.each do |code|

data/Rakefile

@@ -20,7 +20,8 @@ Hoe.spec 'factbook' do
   self.extra_deps = [
     ['logutils' ],
     ['fetcher'],
-    ['nokogiri']
+    ['nokogiri'],
+    ['activerecord']  # NB: will include activesupport,etc.
   ]
 
   self.licenses = ['Public Domain']

data/lib/factbook.rb

@@ -3,6 +3,7 @@
 ## stdlibs
 
 require 'net/http'
+require 'net/https'     ## note: cia factbook requires https
 require 'uri'
 require 'cgi'
 require 'pp'
@@ -18,6 +19,8 @@ require 'logutils'
 require 'fetcher'
 require 'nokogiri'
 
+require 'active_record'     ## add activerecord/db support (NOT optional for now) 
+
 
 # our own code
 
@@ -36,6 +39,10 @@ require 'factbook/comparisons'
 
 require 'factbook/table'    ## e.g. TableReader
 
+require 'factbook/db/schema'   ## database (sql tables) support
+require 'factbook/db/models'
+require 'factbook/db/importer'
+
 
 
 module Factbook

data/lib/factbook/db/importer.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+
+module Factbook
+
+class Importer
+
+def import( page )
+
+  ## note: assumes active connection
+
+  code  = page.info.country_code
+  name  = page.info.country_name
+
+  attribs = {
+   name:       name,
+   area:       sq_km( page.area ),        # e.g. 83,871 sq km       
+   area_land:  sq_km( page.area_land ),   # e.g. 82,445 sq km
+   area_water: sq_km( page.area_water ),  # e.g.  1,426 sq km
+
+   population:         num( page.population ),                   # e.g.  8,665,550 (July 2015 est.)  
+   population_growth:  percent( page.population_growth ),        # e.g.  0.55% (2015 est.)
+   birth_rate:         rate_per_thousand( page.birth_rate ),     # e.g.  9.41 births/1,000 population (2015 est.)
+   death_rate:         rate_per_thousand( page.death_rate ),     # e.g.  9.42 deaths/1,000 population (2015 est.)
+   migration_rate:     rate_per_thousand( page.migration_rate ), # e.g.  5.56 migrant(s)/1,000 population (2015 est.)
+  }
+
+  rec = Fact.find_by( code: code )
+  if rec.nil?   ## create (new) record
+    rec = Fact.new
+    attribs[ :code ] = code
+    puts "create fact record #{code}/#{name}:"
+  else          ## update (exisiting) record
+    puts "update fact record #{code}/#{name}:"
+  end
+  
+  puts "  #{attribs.inspect}"
+  rec.update_attributes!( attribs )
+end
+
+
+def rate_per_thousand( text )
+  # e.g.  9.41 births/1,000 population (2015 est.)
+  #       9.42 deaths/1,000 population (2015 est.)
+  #       5.56 migrant(s)/1,000 population (2015 est.)
+
+  if text =~/([0-9\.]+) [a-z\(\)]+\/1,000/
+    $1.to_f
+  else
+    puts "*** warn: unknown rate <name>/1,000 format (no match): >#{text}<"    
+    nil
+  end
+end
+
+def num( text )
+  # e.g. 8,665,550 (July 2015 est.) 
+
+  if text =~/([0-9,\.]+)/
+    $1.gsub(',', '').to_i   ## note: remove commas (,) if present
+  else
+    puts "*** warn: unknown number format (no match): >#{text}<"
+    nil   ## return nil
+  end
+end
+
+def percent( text )
+  # e.g. 0.55% (2015 est.)
+
+  if text =~/([0-9\.]+)%/
+    $1.to_f
+  else
+    puts "*** warn: unknown percent format (no match): >#{text}<"
+    nil   ## return nil
+  end
+end
+
+def sq_km( text )
+  # e.g. 83,871 sq km
+  ##   todo - check vatican - uses float e.g. 0.44 ?? add support?
+  
+  if text =~/([0-9,\.]+) sq km/
+    $1.gsub(',', '').to_i   ## note: remove commas (,) if present
+  else
+    puts "*** warn: unknown sq km format (no match): >#{text}<"
+    nil   ## return nil
+  end
+end
+
+
+end # class Importer
+
+end # module Factbook
+

data/lib/factbook/db/models.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+module Factbook
+
+
+class Fact < ActiveRecord::Base
+    
+end  # class Fact
+
+
+end # module Factbook

data/lib/factbook/db/schema.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+module Factbook
+
+class CreateDb
+
+def up
+
+ActiveRecord::Schema.define do
+
+create_table :facts do |t|
+  t.string  :code,   null: false   # country code e.g. au
+  t.string  :name,   null: false   # country name e.g. Austria
+
+  t.integer :area        # e.g. 83,871 sq km       
+  t.integer :area_land   # e.g. 82,445 sq km    --use float - why? why not?
+  t.integer :area_water  # e.g.  1,426 sq km
+
+  t.integer :population            # e.g.  8,665,550 (July 2015 est.)  
+  t.float   :population_growth     # e.g.  0.55% (2015 est.)
+  t.float   :birth_rate            # e.g.  9.41 births/1,000 population (2015 est.)
+  t.float   :death_rate            # e.g.  9.42 deaths/1,000 population (2015 est.)
+  t.float   :migration_rate        # e.g.  5.56 migrant(s)/1,000 population (2015 est.)
+
+  t.timestamps
+end
+
+
+end # block Schema.define
+
+end # method up
+
+
+end # class CreateDb
+
+end # module Factbook

data/lib/factbook/page.rb

@@ -29,6 +29,7 @@ class Page
   include LogUtils::Logging
 
   attr_reader :sects    ## "structured" access e.g. sects/subsects/etc.
+  attr_reader :info     ##  meta info e.g. country_code, country_name, region_name, last_updated, etc.
   attr_reader :data     ## "plain" access with vanilla hash
 
 
@@ -49,6 +50,7 @@ class Page
     
     b = Builder.from_string( html )
     @sects = b.sects
+    @info  = b.page_info    ## todo: change b.page_info to info too - why? why not??
 
     @data = {}    
     @sects.each do |sect|
@@ -80,6 +82,49 @@ class Page
     data[key]
   end
 
+  ## add convenience (shortcut) accessors / attributes / fields / getters
+
+  ATTRIBUTES = {
+   'Introduction' => [[:background, 'Background' ]],
+   'Geography'    => [[:area,             'Area', 'total'],    ## convert to number -- why? why not??
+                      [:area_land,        'Area', 'land' ],
+                      [:area_water,       'Area', 'water'],
+                      [:area_note,        'Area', 'note' ],
+                      [:area_comparative, 'Area - comparative'],
+                      [:climate,          'Climate'],
+                      [:terrain,          'Terrain'],
+                      [:elevation_lowest, 'Elevation extremes', 'lowest point'],
+                      [:elevation_highest,'Elevation extremes', 'highest point'],
+                      [:resources,        'Natural resources']],
+  'People and Society' => [[:languages,         'Languages' ],
+                           [:religions,         'Religions' ],
+                           [:population,        'Population' ],
+                           [:population_growth, 'Population growth rate' ],
+                           [:birth_rate,        'Birth rate' ],
+                           [:death_rate,        'Death rate' ],
+                           [:migration_rate,    'Net migration rate' ],
+                           [:major_cities,      'Major urban areas - population' ]],
+  }
+  
+  ATTRIBUTES.each do |section_title, attribs|
+    attribs.each do |attrib|
+      ## e.g.
+      ##    def background()  data['Introduction']['Background']['text']; end  
+      ##    def location()    data['Geography']['Location']['text'];      end
+      ##    etc.
+      if attrib.size == 2
+        define_method attrib[0] do
+          @data.fetch( section_title, {} ).fetch( attrib[1], {} )['text']
+        end
+      else  ## assume size 3 for now
+        define_method attrib[0] do
+          @data.fetch( section_title, {} ).fetch( attrib[1], {} ).fetch( attrib[2], {} )['text']
+        end
+      end
+    end
+  end   
+
+
 private
   def fetch_page( url_string )
 

data/lib/factbook/version.rb

@@ -3,8 +3,8 @@
 module Factbook
 
   MAJOR = 1
-  MINOR = 0
-  PATCH = 1
+  MINOR = 1
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/script/testbr.rb

@@ -8,3 +8,26 @@ require 'factbook'
 page = Factbook::Page.new( 'br' )   # br is the country code for Brazil
 pp page.data                        # pretty print hash
 
+puts "background:"
+pp page.background        ## same as page['Introduction']['Background']['text']
+puts "area:"
+pp page.area              ## same as page['Geography'][''Area']['total']['text']
+puts "area_land:"
+pp page.area_land         ## same as page['Geography'][''Area']['land']['text']
+puts "area_water:"
+pp page.area_water        ## same as page['Geography'][''Area']['water']['text']
+puts "area_note:"
+pp page.area_note         ## same as page['Geography'][''Area']['note']['text']
+puts "area comparative:"
+pp page.area_comparative  ## same as page['Geography']['Area - comparative']['text']
+puts "climate:"
+pp page.climate           ## same as page['Geography']['Climate']['text']
+puts "terrain:"
+pp page.terrain           ## same as page['Geography']['Terrain']['text']
+puts "elevation_lowest:"
+pp page.elevation_lowest  ## same as page['Geography']['Elevation extremes']['lowest point']['text']
+puts "elevation_highest:"
+pp page.elevation_highest ## same as page['Geography']['Elevation extremes']['highest point']['text']
+puts "resources:"
+pp page.resources         ## same as page['Geography'][Natural resources']['text']
+

data/test/test_attribs.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_attribs.rb
+
+require 'helper'
+
+
+class TestAttribs < MiniTest::Test
+
+  def read_test_page( code )
+    html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
+    page = Factbook::Page.new( code, html: html )
+    page    
+  end
+
+  def test_au
+    page = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    ########
+    ## Introduction
+    assert_equal page.background, "Once the center of power for the large Austro-Hungarian Empire, Austria was reduced to a small republic after its defeat in World War I. Following annexation by Nazi Germany in 1938 and subsequent occupation by the victorious Allies in 1945, Austria's status remained unclear for a decade. A State Treaty signed in 1955 ended the occupation, recognized Austria's independence, and forbade unification with Germany. A constitutional law that same year declared the country's \"perpetual neutrality\" as a condition for Soviet military withdrawal. The Soviet Union's collapse in 1991 and Austria's entry into the European Union in 1995 have altered the meaning of this neutrality. A prosperous, democratic country, Austria entered the EU Economic and Monetary Union in 1999."
+    
+    ###########
+    ## Geography
+    assert_equal page.area,       "83,871 sq km"       
+    assert_equal page.area_land,  "82,445 sq km"
+    assert_equal page.area_water, "1,426 sq km"
+    assert_equal page.area_note,  nil
+    assert_equal page.area_comparative, "about the size of South Carolina; slightly more than two-thirds the size of Pennsylvania"
+    assert_equal page.climate,    "temperate; continental, cloudy; cold winters with frequent rain and some snow in lowlands and snow in mountains; moderate summers with occasional showers"  
+    assert_equal page.terrain,    "mostly mountains (Alps) in the west and south; mostly flat or gently sloping along the eastern and northern margins"   
+    assert_equal page.elevation_lowest,   "Neusiedler See 115 m"
+    assert_equal page.elevation_highest,  "Grossglockner 3,798 m"
+    assert_equal page.resources,  "oil, coal, lignite, timber, iron ore, copper, zinc, antimony, magnesite, tungsten, graphite, salt, hydropower"       
+
+    ###################
+    ## People and Society
+    assert_equal page.languages,    "German (official nationwide) 88.6%, Turkish 2.3%, Serbian 2.2%, Croatian (official in Burgenland) 1.6%, other (includes Slovene, official in South Carinthia, and Hungarian, official in Burgenland) 5.3% (2001 est.)"
+    assert_equal page.religions,    "Catholic 73.8% (includes Roman Catholic 73.6%, other Catholic .2%), Protestant 4.9%, Muslim 4.2%, Orthodox 2.2%, other 0.8% (includes other Christian), none 12%, unspecified 2% (2001 est.)"
+    assert_equal page.population,   "8,665,550 (July 2015 est.)"  
+    assert_equal page.population_growth, "0.55% (2015 est.)"
+    assert_equal page.birth_rate,        "9.41 births/1,000 population (2015 est.)"
+    assert_equal page.death_rate,        "9.42 deaths/1,000 population (2015 est.)"
+    assert_equal page.migration_rate,    "5.56 migrant(s)/1,000 population (2015 est.)"
+    assert_equal page.major_cities,      "VIENNA (capital) 1.753 million (2015)"
+
+  end
+
+end # class TestAttribs

data/test/test_convert.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_convert.rb
+
+require 'helper'
+
+
+class TestConvert < MiniTest::Test
+
+  def test_au
+    im = Factbook::Importer.new
+    
+    ###########
+    ## Geography
+    assert_equal 83_871, im.sq_km( "83,871 sq km" )  ## page.area       
+    assert_equal 82_445, im.sq_km( "82,445 sq km" )  ## page.area_land
+    assert_equal  1_426, im.sq_km( "1,426 sq km" )   ## page.area_water
+
+    ###################
+    ## People and Society
+    assert_equal 8_665_550, im.num( "8,665,550 (July 2015 est.)" )  ## page.population  
+    assert_equal 0.55,      im.percent( "0.55% (2015 est.)" ) ## page.population_growth
+    assert_equal 9.41,      im.rate_per_thousand( "9.41 births/1,000 population (2015 est.)" ) ## page.birth_rate
+    assert_equal 9.42,      im.rate_per_thousand( "9.42 deaths/1,000 population (2015 est.)" ) ## page.death_rate
+    assert_equal 5.56,      im.rate_per_thousand( "5.56 migrant(s)/1,000 population (2015 est.)" ) ## page.migration_rate
+  end
+
+end # class TestConvert

data/test/test_importer.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_importer.rb
+
+require 'helper'
+
+
+class TestImporter < MiniTest::Test
+
+  def setup_in_memory_db
+    # Database Setup & Config
+    ActiveRecord::Base.logger = Logger.new( STDOUT )
+    ## ActiveRecord::Base.colorize_logging = false  - no longer exists - check new api/config setting?
+
+    ActiveRecord::Base.establish_connection( adapter:  'sqlite3',
+                                             database: ':memory:' )
+
+    ## build schema
+    Factbook::CreateDb.new.up
+  end
+
+  def read_test_page( code )
+    html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
+    page = Factbook::Page.new( code, html: html )
+    page    
+  end
+
+  def test_au
+    page = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    setup_in_memory_db()
+    
+    im = Factbook::Importer.new
+    im.import( page )
+    
+    rec = Factbook::Fact.find_by! code: 'au'
+
+    ###########
+    ## Geography
+    assert_equal 83_871,    rec.area       
+    assert_equal 82_445,    rec.area_land
+    assert_equal  1_426,    rec.area_water
+
+    ###################
+    ## People and Society
+    assert_equal 8_665_550, rec.population
+    assert_equal 0.55,      rec.population_growth
+    assert_equal 9.41,      rec.birth_rate
+    assert_equal 9.42,      rec.death_rate
+    assert_equal 5.56,      rec.migration_rate
+  end
+
+end # class TestImporter

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: factbook
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-31 00:00:00.000000000 Z
+date: 2015-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -103,6 +117,9 @@ files:
 - lib/factbook/builder_item.rb
 - lib/factbook/codes.rb
 - lib/factbook/comparisons.rb
+- lib/factbook/db/importer.rb
+- lib/factbook/db/models.rb
+- lib/factbook/db/schema.rb
 - lib/factbook/page.rb
 - lib/factbook/sanitizer.rb
 - lib/factbook/sect.rb
@@ -120,10 +137,13 @@ files:
 - test/data/src/au.html
 - test/data/src/be.html
 - test/helper.rb
+- test/test_attribs.rb
 - test/test_builder.rb
 - test/test_codes.rb
 - test/test_comparisons.rb
+- test/test_convert.rb
 - test/test_fields.rb
+- test/test_importer.rb
 - test/test_item_builder.rb
 - test/test_json.rb
 - test/test_page.rb