factbook-readers 0.0.1

data/test/helper.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+## $:.unshift(File.dirname(__FILE__))
+
+## minitest setup
+require 'minitest/autorun'
+
+
+## our own code
+require 'factbook'
+

data/test/test_attribs.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_attribs.rb
+
+require 'helper'
+
+
+class TestAttribs < MiniTest::Test
+
+  def read_test_page( code )
+    html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
+    page = Factbook::Page.new( code, html: html )
+    page
+  end
+
+  def read_test_page_from_json( code )
+    json = File.read( "#{Factbook.root}/test/data/json/#{code}.json" )
+    page = Factbook::Page.new( code, json: json )
+    page
+  end
+
+
+  def test_au_from_html
+    page = read_test_page( 'au' )  # note: use builtin test page (do NOT fetch via internet)
+
+    assert_page_au( page )
+  end
+
+  def xxx_test_au_from_json
+
+    ## todo/fix: check some issue with newlines? when comparing background or something ???
+    page = read_test_page_from_json( 'au' )
+
+    assert_page_au( page )
+  end
+
+private
+  def assert_page_au( page )
+    ########
+    ## Introduction
+    assert_equal page.background, "Once the center of power for the large Austro-Hungarian Empire, Austria was reduced to a small republic after its defeat in World War I. Following annexation by Nazi Germany in 1938 and subsequent occupation by the victorious Allies in 1945, Austria's status remained unclear for a decade. A State Treaty signed in 1955 ended the occupation, recognized Austria's independence, and forbade unification with Germany. A constitutional law that same year declared the country's \"perpetual neutrality\" as a condition for Soviet military withdrawal. The Soviet Union's collapse in 1991 and Austria's entry into the EU in 1995 have altered the meaning of this neutrality. A prosperous, democratic country, Austria entered the EU Economic and Monetary Union in 1999."
+
+    ###########
+    ## Geography
+    assert_equal page.area,       "83,871 sq km"
+    assert_equal page.area_land,  "82,445 sq km"
+    assert_equal page.area_water, "1,426 sq km"
+    assert_equal page.area_note,  nil
+    assert_equal page.area_comparative, "about the size of South Carolina; slightly more than two-thirds the size of Pennsylvania"
+    assert_equal page.climate,    "temperate; continental, cloudy; cold winters with frequent rain and some snow in lowlands and snow in mountains; moderate summers with occasional showers"
+    assert_equal page.terrain,    "mostly mountains (Alps) in the west and south; mostly flat or gently sloping along the eastern and northern margins"
+
+    # was:
+    ## assert_equal page.elevation_lowest,   "Neusiedler See 115 m"
+    ## assert_equal page.elevation_highest,  "Grossglockner 3,798 m"
+    ## new ????
+    ## assert_equal page.elevation_extremes,  "lowest point: Neusiedler See 115 m ++ highest point: Grossglockner 3,798 m"
+
+    assert_equal page.resources,  "oil, coal, lignite, timber, iron ore, copper, zinc, antimony, magnesite, tungsten, graphite, salt, hydropower"
+
+    ###################
+    ## People and Society
+    assert_equal page.languages,    "German (official nationwide) 88.6%, Turkish 2.3%, Serbian 2.2%, Croatian (official in Burgenland) 1.6%, other (includes Slovene, official in South Carinthia, and Hungarian, official in Burgenland) 5.3% (2001 est.)"
+    assert_equal page.religions,    "Catholic 73.8% (includes Roman Catholic 73.6%, other Catholic 0.2%), Protestant 4.9%, Muslim 4.2%, Orthodox 2.2%, other 0.8% (includes other Christian), none 12%, unspecified 2% (2001 est.)"
+    assert_equal page.population,   "8,711,770 (July 2016 est.)"
+    assert_equal page.population_growth, "0.51% (2016 est.)"
+    assert_equal page.birth_rate,        "9.5 births/1,000 population (2016 est.)"
+    assert_equal page.death_rate,        "9.5 deaths/1,000 population (2016 est.)"
+    assert_equal page.migration_rate,    "5.2 migrant(s)/1,000 population (2016 est.)"
+    assert_equal page.major_cities,      "VIENNA (capital) 1.753 million (2015)"
+
+
+    ####################
+    ## Economy
+##    assert_equal page.economy_overview,   "Austria, with its well-developed market economy, skilled labor force, and high standard of living, is closely tied to other EU economies, especially Germany's. Its economy features a large service sector, a relatively sound industrial sector, and a small, but highly developed agricultural sector. Economic growth was anemic at less than 0.5% in 2013 and 2014, and growth in 2015 is not expected to exceed 0.5%. Austria’s 5.6% unemployment rate, while low by European standards, is at an historic high for Austria. Without extensive vocational training programs and generous early retirement, the unemployment rate would be even higher. Public finances have not stabilized even after a 2012 austerity package of expenditure cuts and new revenues. On the contrary, in 2014, the government created a “bad bank” for the troubled nationalized “Hypo Alpe Adria” bank, pushing the budget deficit up by 0.9% of GDP to 2.4% and public debt to 84.5% of the GDP. Although Austria's fiscal position compares favorably with other euro-zone countries, it faces several external risks, such as Austrian banks' continued exposure to Central and Eastern Europe, repercussions from the Hypo Alpe Adria bank collapse, political and economic uncertainties caused by the European sovereign debt crisis, the current crisis in Russia/Ukraine, the recent appreciation of the Swiss Franc, and political developments in Hungary."
+    assert_equal page.gdp_ppp,            "$405.1 billion (2015 est.) ++ $401.6 billion (2014 est.) ++ $400.2 billion (2013 est.)"
+    assert_equal page.gdp_ppp_note,       "data are in 2015 US dollars"
+    assert_equal page.gdp,                 "$374.3 billion (2015 est.)"
+    assert_equal page.gdp_growth,          "0.9% (2015 est.) ++ 0.4% (2014 est.) ++ 0.3% (2013 est.)"
+    assert_equal page.gdp_ppp_capita,      "$47,000 (2015 est.) ++ $47,000 (2014 est.) ++ $47,200 (2013 est.)"
+    assert_equal page.gdp_ppp_capita_note, "data are in 2015 US dollars"
+    assert_equal page.saving,              "25% of GDP (2015 est.) ++ 24.7% of GDP (2014 est.) ++ 25.2% of GDP (2013 est.)"
+  end
+
+end # class TestAttribs

data/test/test_attribs_def.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_attribs_def.rb
+
+require 'helper'
+
+
+class TestAttribsDef < MiniTest::Test
+
+  def test_attribs
+    
+    attribs = Factbook.attributes
+    pp attribs
+    
+    assert true
+  end
+
+end # class TestAttribsDef

data/test/test_builder.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_builder.rb
+
+
+require 'helper'
+
+
+##
+## use/fix: ASCII-8BIT (e.g.keep as is)
+
+
+class TestBuilder < MiniTest::Test
+
+  def test_build
+    
+    ['au','be'].each do |code|
+       ## use/fix: ASCII-8BIT (e.g.keep as is) -???
+       ## fix/todo: use ASCII8BIT/binary reader ??
+      b = Factbook::Builder.from_file( "#{Factbook.root}/test/data/src/#{code}.html" )
+      pp b.sects
+
+      File.open( "./tmp/#{code}.debug.html", 'w' ) do |f|
+        f.write b.html_debug
+      end
+    end
+
+    assert true    ## assume everthing ok
+  end
+
+
+end # class TestBuilder
+

data/test/test_codes.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_codes.rb
+
+
+require 'helper'
+
+
+class TestCodes < MiniTest::Test
+
+
+  def test_codes
+    
+    assert_equal 261, Factbook::CODES.size
+    assert_equal 261, Factbook.codes.size 
+    assert_equal 261, Factbook.codes.to_a.size
+
+
+    assert_equal 195, Factbook.codes.countries.size
+    assert_equal  52, Factbook.codes.dependencies.size
+    assert_equal   5, Factbook.codes.oceans.size
+    assert_equal   1, Factbook.codes.world.size
+    assert_equal   2, Factbook.codes.others.size
+    assert_equal   6, Factbook.codes.misc.size
+
+    assert_equal   8, Factbook.codes.dependencies_us.size
+
+    
+    assert_equal  55, Factbook.codes.europe.size
+    assert_equal   9, Factbook.codes.south_asia.size
+    assert_equal   6, Factbook.codes.central_asia.size
+    assert_equal  22, Factbook.codes.east_n_souteast_asia.size
+    assert_equal  19, Factbook.codes.middle_east.size
+    assert_equal  56, Factbook.codes.africa.size
+    assert_equal   7, Factbook.codes.north_america.size
+    assert_equal  33, Factbook.codes.central_america_n_caribbean.size
+    assert_equal  14, Factbook.codes.south_america.size
+    assert_equal  30, Factbook.codes.australia_oceania.size
+    assert_equal   4, Factbook.codes.antartica.size
+    assert_equal   5, Factbook.codes.region('Oceans').size
+    assert_equal   1, Factbook.codes.region('World').size
+
+    assert_equal  45, Factbook.codes.countries.europe.size
+    
+    assert_equal  Factbook.codes.category('Oceans').size, Factbook.codes.region('Oceans').size
+    assert_equal  Factbook.codes.category('World').size,  Factbook.codes.region('World').size
+
+    
+    assert_equal 261, Factbook.codes.countries.size +
+                      Factbook.codes.others.size +
+                      Factbook.codes.dependencies.size +
+                      Factbook.codes.misc.size +
+                      Factbook.codes.oceans.size +
+                      Factbook.codes.world.size
+
+    assert_equal 261, Factbook.codes.europe.size +
+                      Factbook.codes.south_asia.size +
+                      Factbook.codes.central_asia.size +
+                      Factbook.codes.east_n_souteast_asia.size +
+                      Factbook.codes.middle_east.size +
+                      Factbook.codes.africa.size +
+                      Factbook.codes.north_america.size +
+                      Factbook.codes.central_america_n_caribbean.size +
+                      Factbook.codes.south_america.size +
+                      Factbook.codes.australia_oceania.size +
+                      Factbook.codes.antartica.size +
+                      Factbook.codes.region('Oceans').size +
+                      Factbook.codes.region('World').size
+                    
+  end
+
+end # class TestCodes
+
+

data/test/test_comparisons.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_comparisons.rb
+
+
+require 'helper'
+
+
+class TestComparisons < MiniTest::Test
+
+  def test_comparisons
+    assert_equal 74, Factbook::COMPARISONS.size
+    assert_equal 74, Factbook.comparisons.size 
+    assert_equal 74, Factbook.comparisons.to_a.size                    
+  end
+
+end # class TestComparisons

data/test/test_convert.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_convert.rb
+
+require 'helper'
+
+
+class TestConvert < MiniTest::Test
+
+  def test_au
+    im = Factbook::Importer.new
+    
+    ###########
+    ## Geography
+    assert_equal 83_871, im.sq_km( "83,871 sq km" )  ## page.area       
+    assert_equal 82_445, im.sq_km( "82,445 sq km" )  ## page.area_land
+    assert_equal  1_426, im.sq_km( "1,426 sq km" )   ## page.area_water
+
+    ###################
+    ## People and Society
+    assert_equal 8_665_550, im.num( "8,665,550 (July 2015 est.)" )  ## page.population  
+    assert_equal 0.55,      im.percent( "0.55% (2015 est.)" ) ## page.population_growth
+    assert_equal 9.41,      im.rate_per_thousand( "9.41 births/1,000 population (2015 est.)" ) ## page.birth_rate
+    assert_equal 9.42,      im.rate_per_thousand( "9.42 deaths/1,000 population (2015 est.)" ) ## page.death_rate
+    assert_equal 5.56,      im.rate_per_thousand( "5.56 migrant(s)/1,000 population (2015 est.)" ) ## page.migration_rate
+  end
+
+end # class TestConvert

data/test/test_counter.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_counter.rb
+
+require 'helper'
+
+
+class TestCounter < MiniTest::Test
+
+  def read_test_page( code )
+    html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
+    page = Factbook::Page.new( code, html: html )
+  end
+
+  def test_counter
+    c = Factbook::Counter.new
+ 
+    codes = %w(au be)
+    codes.each do |code|
+      c.count( read_test_page( code ))  # use builtin test page (do NOT fetch via internet)
+    end
+    
+    h = c.data
+    pp h
+    
+    assert true    ## assume everything ok if we get here    
+  end
+
+end # class TestCounter

data/test/test_fields.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_fields.rb
+
+require 'helper'
+
+
+class TestFields < MiniTest::Test
+
+  def read_test_page( code )
+    html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
+    page = Factbook::Page.new( code, html: html )
+    page
+  end
+
+  def test_fields_full
+    ## Factbook::Page.new( 'au', fields: 'full' )
+    page = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    assert_equal '-1.1% of GDP (2015 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
+    assert_equal '0.7%',  page['Economy']['Labor force - by occupation']['agriculture']['text']
+
+    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']['text']
+  end
+
+  def fix_xxx_test_fields_full_w_header
+    page = Factbook::OldPage.new( 'au', header: true, fields: 'full' )
+    page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    assert_equal 'au', page['Header']['code']
+    assert_equal "factbook/#{Factbook::VERSION}", page['Header']['generator']
+
+    assert_equal '-2.4% of GDP (2014 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
+    assert_equal '5.5%',  page['Economy']['Labor force - by occupation']['agriculture']
+
+    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']
+  end
+
+  def fix_xxx_test_fields_std
+    page = Factbook::OldPage.new( 'au' )
+    page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    assert_equal '-2.4% of GDP (2014 est.)', page['econ']['budget_surplus_or_deficit']['text']
+    assert_equal '5.5%',  page['econ']['labor_force_by_occupation']['agriculture']
+
+    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['trans']['ports_and_terminals']['river_ports']
+  end
+
+
+end # class TestFields

data/test/test_importer.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_importer.rb
+
+require 'helper'
+
+
+class TestImporter < MiniTest::Test
+
+  def setup_in_memory_db
+    # Database Setup & Config
+    ActiveRecord::Base.logger = Logger.new( STDOUT )
+    ## ActiveRecord::Base.colorize_logging = false  - no longer exists - check new api/config setting?
+
+    ActiveRecord::Base.establish_connection( adapter:  'sqlite3',
+                                             database: ':memory:' )
+
+    ## build schema
+    Factbook::CreateDb.new.up
+  end
+
+  def read_test_page( code )
+    html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
+    page = Factbook::Page.new( code, html: html )
+    page
+  end
+
+
+  def to_be_done_test_au_fix_me
+    page = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    setup_in_memory_db()
+
+    im = Factbook::Importer.new
+    im.import( page )
+
+    rec = Factbook::Fact.find_by! code: 'au'
+
+    ###########
+    ## Geography
+    assert_equal 83_871,    rec.area
+    assert_equal 82_445,    rec.area_land
+    assert_equal  1_426,    rec.area_water
+
+    ###################
+    ## People and Society
+    assert_equal 8_665_550, rec.population
+    assert_equal 0.55,      rec.population_growth
+    assert_equal 9.41,      rec.birth_rate
+    assert_equal 9.42,      rec.death_rate
+    assert_equal 5.56,      rec.migration_rate
+  end
+
+end # class TestImporter

data/test/test_item_builder.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_item_builder.rb
+
+
+require 'helper'
+
+
+class TestItemBuilder < MiniTest::Test
+
+  def test_location
+
+html =<<EOS
+<div class=category_data>Central Europe, north of Italy and Slovenia</div>
+EOS
+   
+    b = Factbook::ItemBuilder.new( html, 'Location' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_area
+html =<<EOS
+<div><span class=category>total: </span><span class=category_data>83,871 sq km</span></div>
+<div><span class=category>land: </span><span class=category_data>82,445 sq km</span></div>
+<div><span class=category>water: </span><span class=category_data>1,426 sq km</span></div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Area' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_land_use
+html =<<EOS
+<div><span class=category>agricultural land: </span><span class=category_data>38.4%</span></div>
+<div class=category_data>arable land 16.5%; permanent crops 0.8%; permanent pasture 21.1%</div>
+<div><span class=category>forest: </span><span class=category_data>47.2%</span></div>
+<div><span class=category>other: </span><span class=category_data>14.4% (2011 est.)</span></div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Land use' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_contraceptive_prevalence_rate
+html =<<EOS
+<div class=category_data>69.6%</div>
+<div><span class=category>note: </span><span class=category_data>percent of women aged 18-46 (2008/09)</span></div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Contraceptive Prevalence Rate' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_drinking_water_source
+html =<<EOS
+<div><span class=category>improved: </span><span class=category_data></span></div>
+<div class=category_data>urban: 100% of population</div>
+<div class=category_data>rural: 100% of population</div>
+<div class=category_data>total: 100% of population</div>
+<div><span class=category>unimproved: </span><span class=category_data></span></div>
+<div class=category_data>urban: 0% of population</div>
+<div class=category_data>rural: 0% of population</div>
+<div class=category_data>total: 0% of population (2015 est.)</div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Drinking Water Source' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_political_pressure_groups_and_leaders
+html =<<EOS
+<div class=category_data>Austrian Trade Union Federation or OeGB (nominally independent but primarily Social Democratic)</div>
+<div class=category_data>Federal Economic Chamber (OeVP-dominated)</div>
+<div class=category_data>Labor Chamber or AK (Social Democratic-leaning think tank)</div>
+<div class=category_data>OeVP-oriented Association of Austrian Industrialists or IV</div>
+<div class=category_data>Roman Catholic Church, including its chief lay organization, Catholic Action</div>
+<div><span class=category>other: </span><span class=category_data>three composite leagues of the Austrian People's Party or OeVP representing business, labor, farmers, and other nongovernment organizations in the areas of environment and human rights</span></div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Political pressure groups and leaders' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+end # class TestItemBuilder
+