factbook 0.1.3 → 1.0.0

data/test/helper.rb

@@ -3,13 +3,9 @@
 ## $:.unshift(File.dirname(__FILE__))
 
 ## minitest setup
-
-# require 'minitest/unit'
 require 'minitest/autorun'
 
-# include MiniTest::Unit # lets us use TestCase instead of MiniTest::Unit::TestCase
 
 ## our own code
-
 require 'factbook'
 

data/test/test_builder.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_builder.rb
+
+
+require 'helper'
+
+
+##
+## use/fix: ASCII-8BIT (e.g.keep as is)
+
+
+class TestBuilder < MiniTest::Test
+
+  def test_build
+    
+    ['au','be'].each do |cc|
+       ## use/fix: ASCII-8BIT (e.g.keep as is) -???
+       ## fix/todo: use ASCII8BIT/binary reader ??
+      b = Factbook::Builder.from_file( "#{Factbook.root}/test/data/src/#{cc}.html" )
+
+      pp b.page
+      pp b.page.data
+
+      File.open( "./tmp/#{cc}.debug.html", 'w' ) do |f|
+        f.write b.html_debug
+      end
+    end
+
+    assert true    ## assume everthing ok
+  end
+
+
+end # class TestBuilder
+

data/test/test_codes.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_codes.rb
+
+
+require 'helper'
+
+
+class TestCodes < MiniTest::Test
+
+
+  def test_codes
+    
+    assert_equal 261, Factbook::CODES.size
+    assert_equal 261, Factbook.codes.size 
+    assert_equal 261, Factbook.codes.to_a.size
+
+
+    assert_equal 195, Factbook.codes.countries.size
+    assert_equal  52, Factbook.codes.dependencies.size
+    assert_equal   5, Factbook.codes.oceans.size
+    assert_equal   1, Factbook.codes.world.size
+    assert_equal   2, Factbook.codes.others.size
+    assert_equal   6, Factbook.codes.misc.size
+
+    assert_equal   8, Factbook.codes.dependencies_us.size
+
+    
+    assert_equal  55, Factbook.codes.europe.size
+    assert_equal   9, Factbook.codes.south_asia.size
+    assert_equal   6, Factbook.codes.central_asia.size
+    assert_equal  22, Factbook.codes.east_n_souteast_asia.size
+    assert_equal  19, Factbook.codes.middle_east.size
+    assert_equal  56, Factbook.codes.africa.size
+    assert_equal   7, Factbook.codes.north_america.size
+    assert_equal  33, Factbook.codes.central_america_n_caribbean.size
+    assert_equal  14, Factbook.codes.south_america.size
+    assert_equal  30, Factbook.codes.australia_oceania.size
+    assert_equal   4, Factbook.codes.antartica.size
+    assert_equal   5, Factbook.codes.region('Oceans').size
+    assert_equal   1, Factbook.codes.region('World').size
+
+    assert_equal  45, Factbook.codes.countries.europe.size
+    
+    assert_equal  Factbook.codes.category('Oceans').size, Factbook.codes.region('Oceans').size
+    assert_equal  Factbook.codes.category('World').size,  Factbook.codes.region('World').size
+
+    
+    assert_equal 261, Factbook.codes.countries.size +
+                      Factbook.codes.others.size +
+                      Factbook.codes.dependencies.size +
+                      Factbook.codes.misc.size +
+                      Factbook.codes.oceans.size +
+                      Factbook.codes.world.size
+
+    assert_equal 261, Factbook.codes.europe.size +
+                      Factbook.codes.south_asia.size +
+                      Factbook.codes.central_asia.size +
+                      Factbook.codes.east_n_souteast_asia.size +
+                      Factbook.codes.middle_east.size +
+                      Factbook.codes.africa.size +
+                      Factbook.codes.north_america.size +
+                      Factbook.codes.central_america_n_caribbean.size +
+                      Factbook.codes.south_america.size +
+                      Factbook.codes.australia_oceania.size +
+                      Factbook.codes.antartica.size +
+                      Factbook.codes.region('Oceans').size +
+                      Factbook.codes.region('World').size
+                    
+  end
+
+end # class TestCodes
+
+

data/test/test_comparisons.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_comparisons.rb
+
+
+require 'helper'
+
+
+class TestComparisons < MiniTest::Test
+
+  def test_comparisons
+    assert_equal 74, Factbook::COMPARISONS.size
+    assert_equal 74, Factbook.comparisons.size 
+    assert_equal 74, Factbook.comparisons.to_a.size                    
+  end
+
+end # class TestComparisons

data/test/test_fields.rb

@@ -1,44 +1,47 @@
 # encoding: utf-8
 
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_fields.rb
 
 require 'helper'
 
 
-class TestFields < MiniTest::Unit::TestCase
+class TestFields < MiniTest::Test
 
   def read_test_page( code )
-    File.read( "#{Factbook.root}/test/data/countrytemplate_#{code}.html" )
+    b = Factbook::Builder.from_file( "#{Factbook.root}/test/data/src/#{code}.html" )
+    b.page    
   end
 
-  def test_fields_full_w_header
-    page = Factbook::Page.new( 'au', header: true, fields: 'full' )
-    page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
-
-    assert_equal 'au', page['Header']['code']
-    assert_equal "factbook/#{Factbook::VERSION}", page['Header']['generator']
+  def test_fields_full
+    ## Factbook::Page.new( 'au', fields: 'full' )
+    page = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
 
-    assert_equal '-3.1% of GDP (2012 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
-    assert_equal '5.5%',  page['Economy']['Labor force - by occupation']['agriculture']
+    assert_equal '-2.4% of GDP (2014 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
+    assert_equal '5.5%',  page['Economy']['Labor force - by occupation']['agriculture']['text']
 
-    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']
+    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']['text']
   end
 
-
-  def test_fields_full
-    page = Factbook::Page.new( 'au', fields: 'full' )
+  def fix_xxx_test_fields_full_w_header
+    page = Factbook::OldPage.new( 'au', header: true, fields: 'full' )
     page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
 
-    assert_equal '-3.1% of GDP (2012 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
+    assert_equal 'au', page['Header']['code']
+    assert_equal "factbook/#{Factbook::VERSION}", page['Header']['generator']
+
+    assert_equal '-2.4% of GDP (2014 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
     assert_equal '5.5%',  page['Economy']['Labor force - by occupation']['agriculture']
 
     assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']
   end
 
-  def test_fields_std
-    page = Factbook::Page.new( 'au' )
+  def fix_xxx_test_fields_std
+    page = Factbook::OldPage.new( 'au' )
     page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
 
-    assert_equal '-3.1% of GDP (2012 est.)', page['econ']['budget_surplus_or_deficit']['text']
+    assert_equal '-2.4% of GDP (2014 est.)', page['econ']['budget_surplus_or_deficit']['text']
     assert_equal '5.5%',  page['econ']['labor_force_by_occupation']['agriculture']
 
     assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['trans']['ports_and_terminals']['river_ports']

data/test/test_item_builder.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_item_builder.rb
+
+
+require 'helper'
+
+
+class TestItemBuilder < MiniTest::Test
+
+  def test_location
+
+html =<<EOS
+<div class=category_data>Central Europe, north of Italy and Slovenia</div>
+EOS
+   
+    b = Factbook::ItemBuilder.new( html, 'Location' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_area
+html =<<EOS
+<div><span class=category>total: </span><span class=category_data>83,871 sq km</span></div>
+<div><span class=category>land: </span><span class=category_data>82,445 sq km</span></div>
+<div><span class=category>water: </span><span class=category_data>1,426 sq km</span></div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Area' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_land_use
+html =<<EOS
+<div><span class=category>agricultural land: </span><span class=category_data>38.4%</span></div>
+<div class=category_data>arable land 16.5%; permanent crops 0.8%; permanent pasture 21.1%</div>
+<div><span class=category>forest: </span><span class=category_data>47.2%</span></div>
+<div><span class=category>other: </span><span class=category_data>14.4% (2011 est.)</span></div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Land use' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_contraceptive_prevalence_rate
+html =<<EOS
+<div class=category_data>69.6%</div>
+<div><span class=category>note: </span><span class=category_data>percent of women aged 18-46 (2008/09)</span></div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Contraceptive Prevalence Rate' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_drinking_water_source
+html =<<EOS
+<div><span class=category>improved: </span><span class=category_data></span></div>
+<div class=category_data>urban: 100% of population</div>
+<div class=category_data>rural: 100% of population</div>
+<div class=category_data>total: 100% of population</div>
+<div><span class=category>unimproved: </span><span class=category_data></span></div>
+<div class=category_data>urban: 0% of population</div>
+<div class=category_data>rural: 0% of population</div>
+<div class=category_data>total: 0% of population (2015 est.)</div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Drinking Water Source' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+  def test_political_pressure_groups_and_leaders
+html =<<EOS
+<div class=category_data>Austrian Trade Union Federation or OeGB (nominally independent but primarily Social Democratic)</div>
+<div class=category_data>Federal Economic Chamber (OeVP-dominated)</div>
+<div class=category_data>Labor Chamber or AK (Social Democratic-leaning think tank)</div>
+<div class=category_data>OeVP-oriented Association of Austrian Industrialists or IV</div>
+<div class=category_data>Roman Catholic Church, including its chief lay organization, Catholic Action</div>
+<div><span class=category>other: </span><span class=category_data>three composite leagues of the Austrian People's Party or OeVP representing business, labor, farmers, and other nongovernment organizations in the areas of environment and human rights</span></div>
+EOS
+
+    b = Factbook::ItemBuilder.new( html, 'Political pressure groups and leaders' )
+    b.read
+    
+    assert true    ## assume everthing ok
+  end
+
+end # class TestItemBuilder
+

data/test/test_json.rb

@@ -1,10 +1,14 @@
 # encoding: utf-8
 
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_json.rb
+
 
 require 'helper'
 
 
-class TestJson < MiniTest::Unit::TestCase
+class TestJson < MiniTest::Test
 
 
   def test_json
@@ -12,27 +16,20 @@ class TestJson < MiniTest::Unit::TestCase
      
      codes = [ 'au',
                'be',
-               'br',
-               'mx',
-               'ls',
-               'vt',
-               'ee',
-               'xx' ]
+               #'br',
+               #'mx',
+               #'ls',
+               #'vt',
+               #'ee',
+               #'xx'
+              ]
 
      codes.each do |code|
-       page = Factbook::Page.new( code )    
-       page.html = File.read( "#{Factbook.root}/test/data/countrytemplate_#{code}.html" )
-
-       ## print first 600 chars
-       pp page.html[0..600]
-
-       ## save for debuging
-    
-       puts "saving a copy to #{code}.html for debugging"
-       File.open( "tmp/#{code}.html", 'w' ) do |f|
-        f.write page.html 
-       end
+       
+       b = Factbook::Builder.from_file( "#{Factbook.root}/test/data/src/#{code}.html" )
 
+       page = b.page    
+       
        h = page.data
        pp h
     
@@ -44,4 +41,4 @@ class TestJson < MiniTest::Unit::TestCase
      end
   end
 
-end # class TestJson
+end # class TestOldJson

data/test/test_page.rb

@@ -1,32 +1,40 @@
 # encoding: utf-8
 
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_page.rb
+
 
 require 'helper'
 
 
-class TestPage < MiniTest::Unit::TestCase
+class TestPage < MiniTest::Test
 
 
   def test_sects
     pages = [
       [ 'au', 10 ],
       [ 'be', 10 ],
-      [ 'br', 10 ],
-      [ 'ee', 10 ],
-      [ 'mx', 10 ],
-      [ 'xx', 10 ],
-      [ 'ls', 9  ],
-      [ 'vt', 8  ]]
+#      [ 'br', 10 ],
+#      [ 'ee', 10 ],
+#      [ 'mx', 10 ],
+#      [ 'xx', 10 ],
+#      [ 'ls', 9  ],
+#      [ 'vt', 8  ],
+      ]
     
     pages.each do |rec|
         code       = rec[0]
         sects_size = rec[1]
 
-        page = Factbook::Page.new( code )    
-        page.html = File.read( "#{Factbook.root}/test/data/countrytemplate_#{code}.html" )
+        b = Factbook::Builder.from_file( "#{Factbook.root}/test/data/src/#{code}.html" )
+        page = b.page
+       
+        ##  page = Factbook::OldPage.new( code )    
+        ## page.html = File.read( "#{Factbook.root}/test/data/old/countrytemplate_#{code}.html" )
 
         ## print first 600 chars
-        pp page.html[0..600]
+        ## pp page.html[0..600]
 
         assert_equal sects_size, page.sects.size
     end

data/test/test_sanitizer.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_sanitizer.rb
+
+
+require 'helper'
+
+
+class TestSanitizer < MiniTest::Test
+
+  def test_sanitize
+    
+    ['au','be'].each do |cc|
+      ## use/fix: ASCII-8BIT (e.g.keep as is) -???
+      html_ascii = File.read( "#{Factbook.root}/test/data/src/#{cc}.html" )     ## fix/todo: use ASCII8BIT/binary reader ??
+
+      html, info, errors = Factbook::Sanitizer.new.sanitize( html_ascii )
+
+      File.open( "./tmp/#{cc}.profile.html", 'w' ) do |f|
+        f.write "** info:\n"
+        f.write info.inspect + "\n\n"
+        f.write "** errors:\n"
+        f.write errors.inspect + "\n\n"
+        f.write "** html:\n"
+        f.write html
+      end
+    end
+
+    assert true    ## assume everthing ok
+  end
+
+end # class TestSanitizer
+

metadata

@@ -1,71 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: factbook
 version: !ruby/object:Gem::Version
-  version: 0.1.3
-  prerelease: 
+  version: 1.0.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-24 00:00:00.000000000 Z
+date: 2015-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &74365370 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *74365370
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fetcher
-  requirement: &74365080 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *74365080
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &74364810 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *74364810
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &74364520 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
   type: :development
   prerelease: false
-  version_requirements: *74364520
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &74364240 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '3.14'
   type: :development
   prerelease: false
-  version_requirements: *74364240
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.14'
 description: factbook - scripts for the world factbook (get open structured data e.g
   JSON etc.)
 email: openmundi@googlegroups.com
@@ -80,56 +94,61 @@ files:
 - Manifest.txt
 - README.md
 - Rakefile
+- data/codes.csv
+- data/comparisons.csv
 - lib/factbook.rb
+- lib/factbook/builder.rb
+- lib/factbook/builder_item.rb
+- lib/factbook/codes.rb
+- lib/factbook/comparisons.rb
 - lib/factbook/page.rb
+- lib/factbook/sanitizer.rb
 - lib/factbook/sect.rb
+- lib/factbook/subsect.rb
+- lib/factbook/table.rb
+- lib/factbook/utils.rb
+- lib/factbook/utils_info.rb
 - lib/factbook/version.rb
-- test/data/countrytemplate_au.html
-- test/data/countrytemplate_be.html
-- test/data/countrytemplate_br.html
-- test/data/countrytemplate_ee.html
-- test/data/countrytemplate_ls.html
-- test/data/countrytemplate_mx.html
-- test/data/countrytemplate_vt.html
-- test/data/countrytemplate_xx.html
+- test/data/au.html
+- test/data/au.yml
+- test/data/be.html
+- test/data/be.yml
+- test/data/src/au.html
+- test/data/src/be.html
 - test/helper.rb
+- test/test_builder.rb
+- test/test_codes.rb
+- test/test_comparisons.rb
 - test/test_fields.rb
+- test/test_item_builder.rb
 - test/test_json.rb
 - test/test_page.rb
-- test/test_page_old.rb
-- test/test_strip.rb
-- .gemtest
-homepage: https://github.com/worlddb/factbook.ruby
+- test/test_sanitizer.rb
+homepage: https://github.com/worlddb/factbook
 licenses:
 - Public Domain
+metadata: {}
 post_install_message: 
 rdoc_options:
-- --main
+- "--main"
 - README.md
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.17
+rubygems_version: 2.2.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: factbook - scripts for the world factbook (get open structured data e.g JSON
   etc.)
-test_files:
-- test/test_page_old.rb
-- test/test_strip.rb
-- test/test_fields.rb
-- test/test_json.rb
-- test/test_page.rb
+test_files: []