factbook-readers 1.0.1 → 1.1.0

data/lib/factbook-readers/sect.rb

@@ -1,28 +0,0 @@
-
-module Factbook
-
-
-class Sect
-  include LogUtils::Logging
-
-  attr_accessor :title        ## use name instead of title - why? why not?
-  attr_accessor :subsects
-
-  def initialize
-    @subsects = []
-  end
-
-  def data
-    ## convert sects to hash
-    @data = {}
-
-    subsects.each_with_index do |subsect,i|
-      @data[ subsect.title ] = subsect.data
-    end
-    @data
-  end
-
-
-end # class Sect
-
-end # module Factbook

data/lib/factbook-readers/subsect.rb

@@ -1,17 +0,0 @@
-
-module Factbook
-
-
-class Subsect
-  include LogUtils::Logging
-
-  attr_accessor :title        ## use name instead of title - why? why not?
-  attr_accessor :data         ## hash holding data e.g. { 'text' => '...' etc. }
-
-  def initialize
-    @data = {}
-  end
-
-end # class Subsect
-
-end # module Factbook

data/lib/factbook-readers/table.rb

@@ -1,51 +0,0 @@
-
-module Factbook
-
-##
-## make more "generic"  - why? why not?
-##   (re)use for other files ?? move to textutils ??
-
-##
-##  for now reads in rows with values separated by at least 3+ spaces e.g.:
-##   see www.cia.gov/library/publications/the-world-factbook/rankorder/rawdata_2119.txt
-## 1      China                      1,367,485,388
-## 2      India                      1,251,695,584
-## 3      European Union             513,949,445
-## 4      United States              321,368,864
-## 5      Indonesia                  255,993,674
-## 6      Brazil                     204,259,812
-
-
-class TableReader
-  include LogUtils::Logging
-
-
-def initialize( text )
-  @text = text
-end
-
-def read
-  recs = []
-
-  line_no = 0
-  @text.each_line do |line|
-    line_no +=1
-    line = line.strip   ## remove leading and trailing whitespace
-    if line.empty?
-      puts "** skipping empty line #{line_no}"
-      next
-    end
-
-    values = line.split( /[ ]{3,}/ )    ## split three or more spaces - use just two ?? why? why not??
-
-    ## puts line
-    ## pp values
-    recs << values
-  end
-  recs
-end
-
-
-end # class TableReader
-
-end # module Factbook

data/lib/factbook-readers/utils.rb

@@ -1,47 +0,0 @@
-
-module Factbook
-  module Utils
-
-
-def values_to_csv( values )
-  buf = ""
-  values.each_with_index do |value,i|
-     buf << ','  if i > 0    ## add comma (except for first value)
-     ## note: allow optional $ sign e.g. $100,000,000
-     ##  !!!! todo/fix: allow optional minus e.g. -44,000
-     if value =~ /^\$?[1-9][,0-9]+[0-9]$/    ### find a better regex - why? why not??
-       ## check if number e.g. 17,098,242  or $17,098,242
-       ##   remove commas  17098242
-       buf << value.gsub( ',', '' )
-     elsif value.index( ',').nil?
-       ## add as is 1:1 (no commana)
-       buf << value
-     else
-       ## escape comma with double quote
-       #   e.g. Guam, The becomes "Guam, The"
-       buf << '"'
-       buf << value
-       buf << '"'
-     end
-  end
-  buf
-end
-
-
-def data_to_csv( recs, headers )
-  text = ""
-
-  text << values_to_csv( headers )
-  text << "\n"
-
-  recs.each do |rec|
-    text << values_to_csv( rec )
-    text << "\n"
-  end
-
-  text
-end
-
-
-  end   # module Utils
-end     # module Factbook

data/lib/factbook-readers/utils_info.rb

@@ -1,128 +0,0 @@
-
-module Factbook
-  module Utils
-
-#######
-## find meta data (about page info)
-
-
-#### e.g. Page last updated on September 16, 2015
-
-MONTH_EN_TO_S={
-  'January'   => '1',
-  'February'  => '2',
-  'March'     => '3',
-  'April'     => '4',
-  'May'       => '5',
-  'June'      => '6',
-  'July'      => '7',
-  'August'    => '8',
-  'September' => '9',
-  'October'   => '10',
-  'November'  => '11',
-  'December'  => '12'
-}
-
-
-
-##
-#  examples (to match):
-#    Page last updated on November 03, 2016
-#    Page last updated on September 24, 2015
-
-PAGE_LAST_UPDATED_REGEX = /
-                           Page \s last \s updated \s on \s
-                            (?<month_en>[a-z]+) \s
-                            (?<day>\d{1,2}), \s
-                            (?<year>\d{4})
-                          /imx
-
-def find_page_last_updated( html )
-  m = PAGE_LAST_UPDATED_REGEX.match( html )
-  if m
-    pp m
-    month_en = m[:month_en]
-    day      = m[:day]
-    year     = m[:year]
-    puts "** bingo - month #{month_en}, day #{day}, year #{year}"
-
-    month = MONTH_EN_TO_S[ month_en ]
-    date_str = "#{year}-#{month}-#{day}"
-    pp date_str
-    date = Date.strptime( date_str, '%Y-%m-%d' )
-    date
-  else
-    nil
-  end
-end
-
-
-##  fallback: find "standalone" country coude
-## e.g.
-##  ccode='au'
-
-COUNTRY_CODE_REGEX = /ccode='(?<cc>[a-z]+)'/
-
-def find_country_code( html )
-  m = COUNTRY_CODE_REGEX.match( html )
-  if m
-    pp m
-    cc = m[:cc]
-    puts "** bingo - country code #{cc}"
-    cc
-  else
-    nil
-  end
-end
-
-
-##
-## e.g. regioncode="eur"
-##      countrycode="au"
-##      countryname="Austria"
-##      flagsubfield=""
-##      countryaffiliation=""
-##      flagdescription=""
-##      flagdescriptionnote=""
-##      region="Europe"
-##
-##   note: countryaffiliation may be empty
-
-
-
-PAGE_INFO_REGEX = /
-             regioncode=(?<q1>"|')(?<region_code>.+?)\k<q1>
-               \s+
-             countrycode=(?<q2>"|')(?<country_code>.+?)\k<q2>       ## is k<3> backref
-               \s+
-              countryname=(?<q3>"|')(?<country>.+?)\k<q3>
-               \s+
-                [^>]+?  ## allow any attribs (note: non-greedy)
-              countryaffiliation=(?<q4>"|')(?<affiliation>.*?)\k<q4>     ## note: might be empty
-               \s+
-                [^>]+?  ## allow any attribs (note: non-greedy)
-              region=(?<q5>"|')(?<region>.+?)\k<q5>    ## check world - might be empty ?? or for ocean ??
-           /imx
-
-
-def find_page_info( html )
-  m = PAGE_INFO_REGEX.match( html )
-  if m
-    pp m
-
-    h = { country_code:        m[:country_code],
-          country_name:        m[:country],
-          country_affiliation: m[:affiliation],
-          region_code:         m[:region_code],
-          region_name:         m[:region] }
-
-    puts "** bingo - #{h.inspect}"
-    h    ## return hash w/ name-value pairs
-  else
-    nil   ## or return empty struct with nils/empty strings - why?? why not??
-  end
-end
-
-
-  end   # module Utils
-end     # module Factbook

data/test/test_builder.rb

@@ -1,30 +0,0 @@
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_builder.rb
-
-
-require 'helper'
-
-
-
-class TestBuilder < MiniTest::Test
-
-  def test_build
-
-    ['au','be'].each do |code|
-       ## use/fix: ASCII-8BIT (e.g.keep as is) -???
-       ## fix/todo: use ASCII8BIT/binary reader ??
-      b = Factbook::Builder.from_file( "#{Factbook.root}/test/data/src/#{code}.html" )
-      pp b.sects
-
-      File.open( "./tmp/#{code}.debug.html", 'w' ) do |f|
-        f.write b.html_debug
-      end
-    end
-
-    assert true    ## assume everthing ok
-  end
-
-
-end # class TestBuilder
-

data/test/test_codes.rb

@@ -1,72 +0,0 @@
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_codes.rb
-
-
-require 'helper'
-
-
-class TestCodes < MiniTest::Test
-
-
-  def test_codes
-    assert_equal 261, Factbook.codes.size
-    assert_equal 261, Factbook.codes.to_a.size
-
-
-    assert_equal 195, Factbook.codes.countries.size
-    assert_equal  52, Factbook.codes.dependencies.size
-    assert_equal   5, Factbook.codes.oceans.size
-    assert_equal   1, Factbook.codes.world.size
-    assert_equal   2, Factbook.codes.others.size
-    assert_equal   6, Factbook.codes.misc.size
-
-    assert_equal   8, Factbook.codes.dependencies_us.size
-
-
-    assert_equal  55, Factbook.codes.europe.size
-    assert_equal   9, Factbook.codes.south_asia.size
-    assert_equal   6, Factbook.codes.central_asia.size
-    assert_equal  22, Factbook.codes.east_n_souteast_asia.size
-    assert_equal  19, Factbook.codes.middle_east.size
-    assert_equal  56, Factbook.codes.africa.size
-    assert_equal   7, Factbook.codes.north_america.size
-    assert_equal  33, Factbook.codes.central_america_n_caribbean.size
-    assert_equal  14, Factbook.codes.south_america.size
-    assert_equal  30, Factbook.codes.australia_oceania.size
-    assert_equal   4, Factbook.codes.antartica.size
-    assert_equal   5, Factbook.codes.region('Oceans').size
-    assert_equal   1, Factbook.codes.region('World').size
-
-    assert_equal  45, Factbook.codes.countries.europe.size
-
-    assert_equal  Factbook.codes.category('Oceans').size, Factbook.codes.region('Oceans').size
-    assert_equal  Factbook.codes.category('World').size,  Factbook.codes.region('World').size
-
-
-    assert_equal 261, Factbook.codes.countries.size +
-                      Factbook.codes.others.size +
-                      Factbook.codes.dependencies.size +
-                      Factbook.codes.misc.size +
-                      Factbook.codes.oceans.size +
-                      Factbook.codes.world.size
-
-    assert_equal 261, Factbook.codes.europe.size +
-                      Factbook.codes.south_asia.size +
-                      Factbook.codes.central_asia.size +
-                      Factbook.codes.east_n_souteast_asia.size +
-                      Factbook.codes.middle_east.size +
-                      Factbook.codes.africa.size +
-                      Factbook.codes.north_america.size +
-                      Factbook.codes.central_america_n_caribbean.size +
-                      Factbook.codes.south_america.size +
-                      Factbook.codes.australia_oceania.size +
-                      Factbook.codes.antartica.size +
-                      Factbook.codes.region('Oceans').size +
-                      Factbook.codes.region('World').size
-
-  end
-
-end # class TestCodes
-
-

data/test/test_comparisons.rb

@@ -1,16 +0,0 @@
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_comparisons.rb
-
-
-require 'helper'
-
-
-class TestComparisons < MiniTest::Test
-
-  def test_comparisons
-    assert_equal 74, Factbook.comparisons.size
-    assert_equal 74, Factbook.comparisons.to_a.size
-  end
-
-end # class TestComparisons

data/test/test_item_builder.rb

@@ -1,97 +0,0 @@
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_item_builder.rb
-
-
-require 'helper'
-
-
-class TestItemBuilder < MiniTest::Test
-
-  def test_location
-
-html =<<EOS
-<div class=category_data>Central Europe, north of Italy and Slovenia</div>
-EOS
-
-    b = Factbook::ItemBuilder.new( html, 'Location' )
-    b.read
-
-    assert true    ## assume everthing ok
-  end
-
-  def test_area
-html =<<EOS
-<div><span class=category>total: </span><span class=category_data>83,871 sq km</span></div>
-<div><span class=category>land: </span><span class=category_data>82,445 sq km</span></div>
-<div><span class=category>water: </span><span class=category_data>1,426 sq km</span></div>
-EOS
-
-    b = Factbook::ItemBuilder.new( html, 'Area' )
-    b.read
-
-    assert true    ## assume everthing ok
-  end
-
-  def test_land_use
-html =<<EOS
-<div><span class=category>agricultural land: </span><span class=category_data>38.4%</span></div>
-<div class=category_data>arable land 16.5%; permanent crops 0.8%; permanent pasture 21.1%</div>
-<div><span class=category>forest: </span><span class=category_data>47.2%</span></div>
-<div><span class=category>other: </span><span class=category_data>14.4% (2011 est.)</span></div>
-EOS
-
-    b = Factbook::ItemBuilder.new( html, 'Land use' )
-    b.read
-
-    assert true    ## assume everthing ok
-  end
-
-  def test_contraceptive_prevalence_rate
-html =<<EOS
-<div class=category_data>69.6%</div>
-<div><span class=category>note: </span><span class=category_data>percent of women aged 18-46 (2008/09)</span></div>
-EOS
-
-    b = Factbook::ItemBuilder.new( html, 'Contraceptive Prevalence Rate' )
-    b.read
-
-    assert true    ## assume everthing ok
-  end
-
-  def test_drinking_water_source
-html =<<EOS
-<div><span class=category>improved: </span><span class=category_data></span></div>
-<div class=category_data>urban: 100% of population</div>
-<div class=category_data>rural: 100% of population</div>
-<div class=category_data>total: 100% of population</div>
-<div><span class=category>unimproved: </span><span class=category_data></span></div>
-<div class=category_data>urban: 0% of population</div>
-<div class=category_data>rural: 0% of population</div>
-<div class=category_data>total: 0% of population (2015 est.)</div>
-EOS
-
-    b = Factbook::ItemBuilder.new( html, 'Drinking Water Source' )
-    b.read
-
-    assert true    ## assume everthing ok
-  end
-
-  def test_political_pressure_groups_and_leaders
-html =<<EOS
-<div class=category_data>Austrian Trade Union Federation or OeGB (nominally independent but primarily Social Democratic)</div>
-<div class=category_data>Federal Economic Chamber (OeVP-dominated)</div>
-<div class=category_data>Labor Chamber or AK (Social Democratic-leaning think tank)</div>
-<div class=category_data>OeVP-oriented Association of Austrian Industrialists or IV</div>
-<div class=category_data>Roman Catholic Church, including its chief lay organization, Catholic Action</div>
-<div><span class=category>other: </span><span class=category_data>three composite leagues of the Austrian People's Party or OeVP representing business, labor, farmers, and other nongovernment organizations in the areas of environment and human rights</span></div>
-EOS
-
-    b = Factbook::ItemBuilder.new( html, 'Political pressure groups and leaders' )
-    b.read
-
-    assert true    ## assume everthing ok
-  end
-
-end # class TestItemBuilder
-