factbook-readers 1.0.1 → 1.1.0

data/lib/factbook-readers/page_info.rb

@@ -1,11 +1,23 @@
 
 module Factbook
 
+## todo/check: change to PageMeta/ProfileMeta
+##   or Page/ProfileMetaInfo or MetaInfo or Meta or such - why? why not?
+
+
+### was before 2021 (when using html page scrapping)
+# PageInfo = Struct.new( :country_code,
+#                          :country_name,
+#                          :country_affiliation,
+#                          :region_code,
+#                          :region_name,
+#                          :last_updated )
+
 PageInfo = Struct.new( :country_code,
                        :country_name,
-                       :country_affiliation,
-                       :region_code,
                        :region_name,
-                       :last_updated )
+                       :published,   ## note: published is NOT before updated (like an alias for created) BUT is often older/later than updated - why!?
+                       :updated )
+
 
 end # module Factbook

data/lib/factbook-readers/version.rb

@@ -3,8 +3,8 @@ module Factbook
 module  Module
 module   Readers
   MAJOR = 1
-  MINOR = 0
-  PATCH = 1
+  MINOR = 1
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/helper.rb

@@ -1,3 +1,6 @@
+$LOAD_PATH.unshift( "../factbook-codes/lib" )
+$LOAD_PATH.unshift( "../factbook-fields/lib" )
+
 ## minitest setup
 require 'minitest/autorun'
 

data/test/test_counter.rb

@@ -8,19 +8,22 @@ require 'helper'
 class TestCounter < MiniTest::Test
 
   def read_test_page( code )
-    html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
-    page = Factbook::Page.new( code, html: html )
+    path = "#{Factbook::Module::Readers.root}/../testdata/json/#{code}.json"
+    page = Factbook::Page.read_json( path )
+    page
   end
 
+
   def test_counter
-    c = Factbook::Counter.new
+    counter = Factbook::Counter.new
 
-    codes = %w(au be)
+    codes = ['au', 'be']
     codes.each do |code|
-      c.count( read_test_page( code ))  # use builtin test page (do NOT fetch via internet)
+      page = read_test_page( code )
+      counter.count( code, page )  # use builtin test page (do NOT fetch via internet)
     end
 
-    h = c.data
+    h = counter.data
     pp h
 
     assert true    ## assume everything ok if we get here

data/test/test_download.rb

@@ -0,0 +1,27 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_download.rb
+
+
+require 'helper'
+
+
+class TestDownload < MiniTest::Test
+
+  def test_pages
+
+    ['au','be'].each do |code|
+
+      page = Factbook::Page.download( code, cache: true )
+
+      File.open( "./tmp/#{code}.debug.json", 'w:utf-8' ) do |f|
+        f.write JSON.pretty_generate( page.to_h )
+      end
+    end
+
+    assert true    ## assume everthing ok
+  end
+
+
+end # class TestDownload
+

data/test/test_fields.rb

@@ -8,43 +8,60 @@ require 'helper'
 class TestFields < MiniTest::Test
 
   def read_test_page( code )
-    html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
-    page = Factbook::Page.new( code, html: html )
+    path = "#{Factbook::Module::Readers.root}/../testdata/json/#{code}.json"
+    page = Factbook::Page.read_json( path )
     page
   end
 
-  def test_fields_full
-    ## Factbook::Page.new( 'au', fields: 'full' )
-    page = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
 
-    assert_equal '-1.1% of GDP (2015 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
-    assert_equal '0.7%',  page['Economy']['Labor force - by occupation']['agriculture']['text']
+  def test_fields_au
+    page = read_test_page( 'au' )  # note: use builtin test page (do NOT fetch via internet)
 
-    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']['text']
-  end
-
-  def fix_xxx_test_fields_full_w_header
-    page = Factbook::OldPage.new( 'au', header: true, fields: 'full' )
-    page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
-
-    assert_equal 'au', page['Header']['code']
-    assert_equal "factbook/#{Factbook::VERSION}", page['Header']['generator']
+    assert_equal '-0.7% (of GDP) (2017 est.)',
+                  page['Economy']['Budget surplus (+) or deficit (-)']['text']
+    assert_equal '0.7%',
+                  page['Economy']['Labor force - by occupation']['agriculture']['text']
 
-    assert_equal '-2.4% of GDP (2014 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
-    assert_equal '5.5%',  page['Economy']['Labor force - by occupation']['agriculture']
-
-    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']
+    assert_equal 'Enns, Krems, Linz, Vienna (Danube)',
+                 page['Transportation']['Ports and terminals']['river port(s)']['text']
   end
 
-  def fix_xxx_test_fields_std
-    page = Factbook::OldPage.new( 'au' )
-    page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
-
-    assert_equal '-2.4% of GDP (2014 est.)', page['econ']['budget_surplus_or_deficit']['text']
-    assert_equal '5.5%',  page['econ']['labor_force_by_occupation']['agriculture']
 
-    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['trans']['ports_and_terminals']['river_ports']
+  def strip_tags( text )
+    ## simple quick and dirty helper
+    text = text.gsub( '<p>', '' )
+    text = text.gsub( '</p>', '' )
+    text = text.gsub( '<strong>', '' )
+    text = text.gsub( '</strong>', '' )
+    text
   end
 
+  def test_fields_br
+    ## check fields from readme
+    page = read_test_page( 'br' )  # note: use builtin test page (do NOT fetch via internet)
+
+    assert strip_tags( page['Introduction']['Background']['text'] ).start_with?(
+              'Following more than three centuries' )
+    assert_equal '8,515,770 sq km',
+                 page['Geography']['Area']['total']['text']
+    assert_equal '8,358,140 sq km',
+                 page['Geography']['Area']['land']['text']
+    assert_equal '157,630 sq km',
+                 page['Geography']['Area']['water']['text']
+    assert strip_tags( page['Geography']['Area']['note'] ).start_with?(
+         'note: includes Arquipelago de Fernando de Noronha, Atol das Rocas,' )
+    assert_equal 'slightly smaller than the US',
+                 page['Geography']['Area - comparative']['text']
+    assert_equal 'mostly tropical, but temperate in south',
+                 page['Geography']['Climate']['text']
+    assert page['Geography']['Terrain']['text'].start_with?(
+         'mostly flat to rolling lowlands in north;' )
+    assert_equal 'Atlantic Ocean 0 m',
+                 page['Geography']['Elevation']['lowest point']['text']
+    assert_equal 'Pico da Neblina 2,994 m',
+                 page['Geography']['Elevation']['highest point']['text']
+    assert page['Geography']['Natural resources']['text'].start_with?(
+         'alumina, bauxite, beryllium, gold, iron ore, manganese, nickel,' )
+  end
 
 end # class TestFields

data/test/test_json.rb

@@ -15,7 +15,7 @@ class TestJson < MiniTest::Test
      codes = [ 'au',
                'ag',
                'be',
-               #'br',
+               'br',
                #'mx',
                #'ls',
                #'vt',
@@ -25,10 +25,10 @@ class TestJson < MiniTest::Test
 
      codes.each do |code|
 
-       html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
-       page = Factbook::Page.new( code, html: html )
+       path = "#{Factbook::Module::Readers.root}/../testdata/json/#{code}.json"
+       page = Factbook::Page.read_json( path )
 
-       h = page.data
+       h = page.to_h
        pp h
 
        ### save to json

data/test/test_page.rb

@@ -9,11 +9,11 @@ require 'helper'
 class TestPage < MiniTest::Test
 
 
-  def test_sects
+  def test_pages
     pages = [
-      [ 'au', 10 ],
-      [ 'be', 10 ],
-#      [ 'br', 10 ],
+      [ 'au', 11 ],
+      [ 'be', 11 ],
+      [ 'br', 10 ],
 #      [ 'ee', 10 ],
 #      [ 'mx', 10 ],
 #      [ 'xx', 10 ],
@@ -23,12 +23,12 @@ class TestPage < MiniTest::Test
 
     pages.each do |rec|
         code       = rec[0]
-        sects_size = rec[1]
+        size       = rec[1]
 
-        html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
-        page = Factbook::Page.new( code, html: html )
+        path = "#{Factbook::Module::Readers.root}/../testdata/json/#{code}.json"
+        page = Factbook::Page.read_json( path )
 
-        assert_equal sects_size, page.sects.size
+        assert_equal size, page.size
     end
   end
 

data/test/test_version.rb

@@ -0,0 +1,15 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_version.rb
+
+require 'helper'
+
+
+class TestVersion < MiniTest::Test
+
+  def test_version
+    pp Factbook::Module::Readers.root
+    pp Factbook::Module::Readers.banner
+  end
+
+end # class TestVersion

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: factbook-readers
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Gerald Bauer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-27 00:00:00.000000000 Z
+date: 2021-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logutils
+  name: factbook-codes
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: csvreader
+  name: factbook-fields
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -52,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -114,47 +100,25 @@ files:
 - Manifest.txt
 - README.md
 - Rakefile
-- data/categories.csv
-- data/codes.csv
-- data/codesxref.csv
-- data/comparisons.csv
 - lib/factbook-readers.rb
-- lib/factbook-readers/builder.rb
-- lib/factbook-readers/builder_item.rb
-- lib/factbook-readers/builder_json.rb
-- lib/factbook-readers/codes.rb
-- lib/factbook-readers/comparisons.rb
+- lib/factbook-readers/convert.rb
 - lib/factbook-readers/counter.rb
-- lib/factbook-readers/normalize.rb
 - lib/factbook-readers/page.rb
 - lib/factbook-readers/page_info.rb
-- lib/factbook-readers/reader_json.rb
-- lib/factbook-readers/sanitizer.rb
-- lib/factbook-readers/sect.rb
-- lib/factbook-readers/subsect.rb
-- lib/factbook-readers/table.rb
-- lib/factbook-readers/utils.rb
-- lib/factbook-readers/utils_info.rb
 - lib/factbook-readers/version.rb
 - lib/factbook/readers.rb
 - test/helper.rb
-- test/test_builder.rb
-- test/test_codes.rb
-- test/test_comparisons.rb
 - test/test_counter.rb
+- test/test_download.rb
 - test/test_fields.rb
-- test/test_item_builder.rb
 - test/test_json.rb
-- test/test_json_builder.rb
-- test/test_normalize.rb
 - test/test_page.rb
-- test/test_sanitizer.rb
-- test/test_sanitizer_regex.rb
+- test/test_version.rb
 homepage: https://github.com/factbook/factbook
 licenses:
 - Public Domain
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -171,9 +135,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: factbook-readers - turn thee world factbook country profile pages into open
   structured data e.g JSON

data/data/categories.csv

@@ -1,164 +0,0 @@
-Num,Category,Name,Notes
-,Introduction,Background,
-,Geography,Location,
-,Geography,Geographic coordinates,
-,Geography,Map references,
-,Geography,Area,
-,Geography,Area - comparative,
-,Geography,Land boundaries,
-,Geography,Coastline,
-,Geography,Maritime claims,
-,Geography,Climate,
-,Geography,Terrain,
-,Geography,Elevation extremes,
-,Geography,Natural resources,
-,Geography,Land use,
-,Geography,Irrigated land,
-,Geography,Total renewable water resources,
-,Geography,Freshwater withdrawal (domestic/industrial/agricultural),
-,Geography,Natural hazards,
-,Geography,Environment - current issues,
-,Geography,Environment - international agreements,
-,Geography,Geography - note,
-,People and Society,Nationality,
-,People and Society,Ethnic groups,
-,People and Society,Languages,
-,People and Society,Religions,
-,People and Society,Population,
-,People and Society,Age structure,
-,People and Society,Dependency ratios,
-,People and Society,Median age,
-,People and Society,Population growth rate,
-,People and Society,Birth rate,
-,People and Society,Death rate,
-,People and Society,Net migration rate,
-,People and Society,Urbanization,
-,People and Society,Major urban areas - population,
-,People and Society,Sex ratio,
-,People and Society,Infant mortality rate,
-,People and Society,Life expectancy at birth,
-,People and Society,Total fertility rate,
-,People and Society,Contraceptive prevalence rate,
-,People and Society,Health expenditures,
-,People and Society,Physicians density,
-,People and Society,Hospital bed density,
-,People and Society,Drinking water source,
-,People and Society,Sanitation facility access,
-,People and Society,HIV/AIDS - adult prevalence rate,
-,People and Society,HIV/AIDS - people living with HIV/AIDS,
-,People and Society,HIV/AIDS - deaths,
-,People and Society,Obesity - adult prevalence rate,
-,People and Society,Education expenditures,
-,People and Society,School life expectancy (primary to tertiary education),
-,People and Society,"Unemployment, youth ages 15-24",
-,Government,Country name,
-,Government,Government type,
-,Government,Capital,
-,Government,Administrative divisions,
-,Government,Independence,
-,Government,National holiday,
-,Government,Constitution,
-,Government,Legal system,
-,Government,International law organization participation,
-,Government,Suffrage,
-,Government,Executive branch,
-,Government,Legislative branch,
-,Government,Judicial branch,
-,Government,Political parties and leaders,
-,Government,Political pressure groups and leaders,
-,Government,International organization participation,
-,Government,Diplomatic representation in the US,
-,Government,Diplomatic representation from the US,
-,Government,Flag description,
-,Government,National symbol(s),
-,Government,National anthem,
-,Economy,Economy - overview,
-,Economy,GDP (purchasing power parity),
-,Economy,GDP (official exchange rate),
-,Economy,GDP - real growth rate,
-,Economy,GDP - per capita (PPP),
-,Economy,Gross national saving,
-,Economy,"GDP - composition, by end use",
-,Economy,"GDP - composition, by sector of origin",
-,Economy,Agriculture - products,
-,Economy,Industries,
-,Economy,Industrial production growth rate,
-,Economy,Labor force,
-,Economy,Labor force - by occupation,
-,Economy,Unemployment rate,
-,Economy,Population below poverty line,
-,Economy,Household income or consumption by percentage share,
-,Economy,Distribution of family income - Gini index,
-,Economy,Budget,
-,Economy,Taxes and other revenues,
-,Economy,Budget surplus (+) or deficit (-),
-,Economy,Public debt,
-,Economy,Fiscal year,
-,Economy,Inflation rate (consumer prices),
-,Economy,Commercial bank prime lending rate,
-,Economy,Stock of narrow money,
-,Economy,Stock of broad money,
-,Economy,Stock of domestic credit,
-,Economy,Market value of publicly traded shares,
-,Economy,Current account balance,
-,Economy,Exports,
-,Economy,Exports - commodities,
-,Economy,Exports - partners,
-,Economy,Imports,
-,Economy,Imports - commodities,
-,Economy,Imports - partners,
-,Economy,Reserves of foreign exchange and gold,
-,Economy,Debt - external,
-,Economy,Stock of direct foreign investment - at home,
-,Economy,Stock of direct foreign investment - abroad,
-,Economy,Exchange rates,
-,Energy,Electricity - production,
-,Energy,Electricity - consumption,
-,Energy,Electricity - exports,
-,Energy,Electricity - imports,
-,Energy,Electricity - installed generating capacity,
-,Energy,Electricity - from fossil fuels,
-,Energy,Electricity - from nuclear fuels,
-,Energy,Electricity - from hydroelectric plants,
-,Energy,Electricity - from other renewable sources,
-,Energy,Crude oil - production,
-,Energy,Crude oil - exports,
-,Energy,Crude oil - imports,
-,Energy,Crude oil - proved reserves,
-,Energy,Refined petroleum products - production,
-,Energy,Refined petroleum products - consumption,
-,Energy,Refined petroleum products - exports,
-,Energy,Refined petroleum products - imports,
-,Energy,Natural gas - production,
-,Energy,Natural gas - consumption,
-,Energy,Natural gas - exports,
-,Energy,Natural gas - imports,
-,Energy,Natural gas - proved reserves,
-,Energy,Carbon dioxide emissions from consumption of energy,
-,Communications,Telephones - fixed lines,
-,Communications,Telephones - mobile cellular,
-,Communications,Telephone system,
-,Communications,Broadcast media,
-,Communications,Radio broadcast stations,
-,Communications,Television broadcast stations,
-,Communications,Internet country code,
-,Communications,Internet users,
-,Transportation,Airports,
-,Transportation,Airports - with paved runways,
-,Transportation,Airports - with unpaved runways,
-,Transportation,Heliports,
-,Transportation,Pipelines,
-,Transportation,Railways,
-,Transportation,Roadways,
-,Transportation,Waterways,
-,Transportation,Merchant marine,
-,Transportation,Ports and terminals,
-,Military,Military branches,
-,Military,Military service age and obligation,
-,Military,Manpower available for military service,
-,Military,Manpower fit for military service,
-,Military,Manpower reaching militarily significant age annually,
-,Military,Military expenditures,
-,Transnational Issues,Disputes - international,
-,Transnational Issues,Refugees and internally displaced persons,
-,Transnational Issues,Illicit drugs,