factbook-readers 0.0.1 → 1.0.0

data/lib/factbook-readers/sanitizer.rb

@@ -64,12 +64,6 @@ def find_country_profile( html )
   end
 
 
-  ## note: replace all non-breaking spaces with spaces for now
-  ##  see fr (france) in political parties section for example
-  html = html.gsub( " ", ' ' )
-
-
-
   doc = Nokogiri::HTML( html )
 
   ul = doc.css( 'ul.expandcollapse' )[0]
@@ -77,30 +71,8 @@ def find_country_profile( html )
   puts ul.to_html[0..100]
 
 
-
-  ## note: special case cc uses h2 instead of div block
-  ##  <h2 class="question cam_med" sectiontitle="Introduction" ccode="cc"
-  ##         style="border-bottom: 2px solid white; cursor: pointer;">
-  ##         Introduction ::  <span class="region">CURACAO </span>
-  ##   </h2>
-  ##   is old format !!!!
-  ##   cc - CURACAO
-  ##  http headers says - last-modified: Wed, 14 Nov 2018 14:09:28 GMT
-  ##   page says - PAGE LAST UPDATED ON MARCH 14, 2018
-  ##    wait for new version to be generated / pushed!!!
-
-  ## check for old format if h2 are present
-  h2s = ul.css( 'h2' )
-  if h2s.size > 0
-    puts "  !! WARN: found #{h2s.size} h2(s) - assume old format - sorry - must wait for update!!!"
-    ## return empty html string - why? why not?
-    return  ''
-  end
-
-
   ###
   ## sanitize
-
   ## remove link items
   ##   assume two <li>s are a section
 
@@ -204,12 +176,12 @@ end
 #  <span class="subfield-date" aria-label="Date of information: 2018">(2018)</span>
 #
 #  remove aria labels
-ARIA_ATTR_REGEX = /\s*
+ARIA_ATTR_RE = /\s*
                      aria-label=('|").+?\1     ## note: use non-greedy match e.g. .+?
                   /xim    ## do NOT allow multi-line - why? why not?
 
 ## find double breaks e.g. <br><br>
-BR_BR_REGEX = /(<br> \s* <br>)
+BR_BR_RE = /(<br> \s* <br>)
               /xim   ## do NOT allow multi-line - why? why not?
 
 
@@ -252,6 +224,12 @@ def sanitize_data( el, title: )
   ## note: keep container div!! just replace inner html!!!
   ##  note: right strip all trailing spaces/newlines for now
   ##        plus add back a single one for pretty printing
+
+  ## note: replace all non-breaking spaces with spaces for now
+  ##  see fr (france) in political parties section for example
+  ##  todo/check/fix:  check if we need to use unicode char!! and NOT html entity
+  inner_html = inner_html.gsub( "&nbsp;", ' ' )
+
   el.inner_html = inner_html.rstrip + "\n"
 
   # finally - convert back to html (string)
@@ -259,14 +237,14 @@ def sanitize_data( el, title: )
 
 
 
-  html = html.gsub( ARIA_ATTR_REGEX ) do |m|
+  html = html.gsub( ARIA_ATTR_RE ) do |m|
     ## do not report / keep silent for now
     ## puts "in >#{title}< remove aria-label attr:"
     ## puts "#{m}"
     ''
   end
 
-  html = html.gsub( BR_BR_REGEX ) do |m|
+  html = html.gsub( BR_BR_RE ) do |m|
     puts "in >#{title}< squish two <br>s into one:"
     puts "#{m}"
     '<br>'
@@ -280,11 +258,12 @@ def sanitize_data( el, title: )
 
   ## cleanup/remove ++   before subfield e.g.
   ##  of: ++   => of:    or such
+  ##
+  ##  todo/fix: add negative lookahead e.g. not another + to be more specific!!
   html = html.gsub( %r{
                        (?<=([a-z]:)|(:</span>))  # note: use zero-length positive lookbehind
                           \s+
-                          \+{2}
-                          \s+}xim ) do |m|
+                          \+{2}}xim ) do |m|
      puts "in >#{title} remove ++ before <field>: marker:"
      puts "#{m}"
     ' '

data/lib/factbook-readers/version.rb

@@ -2,9 +2,9 @@
 module Factbook
 module  Module
 module   Readers
-  MAJOR = 0
+  MAJOR = 1
   MINOR = 0
-  PATCH = 1
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/helper.rb

@@ -1,11 +1,7 @@
-# encoding: utf-8
-
-## $:.unshift(File.dirname(__FILE__))
-
 ## minitest setup
 require 'minitest/autorun'
 
 
 ## our own code
-require 'factbook'
+require 'factbook/readers'
 

data/test/test_builder.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_builder.rb
@@ -8,14 +6,11 @@
 require 'helper'
 
 
-##
-## use/fix: ASCII-8BIT (e.g.keep as is)
-
 
 class TestBuilder < MiniTest::Test
 
   def test_build
-    
+
     ['au','be'].each do |code|
        ## use/fix: ASCII-8BIT (e.g.keep as is) -???
        ## fix/todo: use ASCII8BIT/binary reader ??

data/test/test_codes.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_codes.rb
@@ -12,9 +10,7 @@ class TestCodes < MiniTest::Test
 
 
   def test_codes
-    
-    assert_equal 261, Factbook::CODES.size
-    assert_equal 261, Factbook.codes.size 
+    assert_equal 261, Factbook.codes.size
     assert_equal 261, Factbook.codes.to_a.size
 
 
@@ -27,7 +23,7 @@ class TestCodes < MiniTest::Test
 
     assert_equal   8, Factbook.codes.dependencies_us.size
 
-    
+
     assert_equal  55, Factbook.codes.europe.size
     assert_equal   9, Factbook.codes.south_asia.size
     assert_equal   6, Factbook.codes.central_asia.size
@@ -43,11 +39,11 @@ class TestCodes < MiniTest::Test
     assert_equal   1, Factbook.codes.region('World').size
 
     assert_equal  45, Factbook.codes.countries.europe.size
-    
+
     assert_equal  Factbook.codes.category('Oceans').size, Factbook.codes.region('Oceans').size
     assert_equal  Factbook.codes.category('World').size,  Factbook.codes.region('World').size
 
-    
+
     assert_equal 261, Factbook.codes.countries.size +
                       Factbook.codes.others.size +
                       Factbook.codes.dependencies.size +
@@ -68,7 +64,7 @@ class TestCodes < MiniTest::Test
                       Factbook.codes.antartica.size +
                       Factbook.codes.region('Oceans').size +
                       Factbook.codes.region('World').size
-                    
+
   end
 
 end # class TestCodes

data/test/test_comparisons.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_comparisons.rb
@@ -11,9 +9,8 @@ require 'helper'
 class TestComparisons < MiniTest::Test
 
   def test_comparisons
-    assert_equal 74, Factbook::COMPARISONS.size
-    assert_equal 74, Factbook.comparisons.size 
-    assert_equal 74, Factbook.comparisons.to_a.size                    
+    assert_equal 74, Factbook.comparisons.size
+    assert_equal 74, Factbook.comparisons.to_a.size
   end
 
 end # class TestComparisons

data/test/test_counter.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_counter.rb
@@ -16,16 +14,16 @@ class TestCounter < MiniTest::Test
 
   def test_counter
     c = Factbook::Counter.new
- 
+
     codes = %w(au be)
     codes.each do |code|
       c.count( read_test_page( code ))  # use builtin test page (do NOT fetch via internet)
     end
-    
+
     h = c.data
     pp h
-    
-    assert true    ## assume everything ok if we get here    
+
+    assert true    ## assume everything ok if we get here
   end
 
 end # class TestCounter

data/test/test_fields.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_fields.rb

data/test/test_item_builder.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_item_builder.rb
@@ -15,10 +13,10 @@ class TestItemBuilder < MiniTest::Test
 html =<<EOS
 <div class=category_data>Central Europe, north of Italy and Slovenia</div>
 EOS
-   
+
     b = Factbook::ItemBuilder.new( html, 'Location' )
     b.read
-    
+
     assert true    ## assume everthing ok
   end
 
@@ -31,7 +29,7 @@ EOS
 
     b = Factbook::ItemBuilder.new( html, 'Area' )
     b.read
-    
+
     assert true    ## assume everthing ok
   end
 
@@ -45,7 +43,7 @@ EOS
 
     b = Factbook::ItemBuilder.new( html, 'Land use' )
     b.read
-    
+
     assert true    ## assume everthing ok
   end
 
@@ -57,7 +55,7 @@ EOS
 
     b = Factbook::ItemBuilder.new( html, 'Contraceptive Prevalence Rate' )
     b.read
-    
+
     assert true    ## assume everthing ok
   end
 
@@ -75,7 +73,7 @@ EOS
 
     b = Factbook::ItemBuilder.new( html, 'Drinking Water Source' )
     b.read
-    
+
     assert true    ## assume everthing ok
   end
 
@@ -91,7 +89,7 @@ EOS
 
     b = Factbook::ItemBuilder.new( html, 'Political pressure groups and leaders' )
     b.read
-    
+
     assert true    ## assume everthing ok
   end
 

data/test/test_json.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_json.rb
@@ -35,7 +33,7 @@ class TestJson < MiniTest::Test
 
        ### save to json
        puts "saving a copy to #{code}.json for debugging"
-       File.open( "tmp/#{code}.json", 'w' ) do |f|
+       File.open( "tmp/#{code}.json", 'w:utf-8' ) do |f|
          f.write JSON.pretty_generate( h )
          ## f.write page.to_json
       end

data/test/test_json_builder.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_json_builder.rb
@@ -13,7 +11,7 @@ class TestJsonBuilder < MiniTest::Test
   def test_read
     code = 'au'
     b = Factbook::JsonBuilder.from_file( "#{Factbook.root}/test/data/json/#{code}.json" )
-    
+
     assert_equal 10, b.sects.size
     assert_equal  1, b.sects[0].subsects.size   ## e.g. Introduction/Background
     assert_equal 'Central Europe, north of Italy and Slovenia', b.json['Geography']['Location']['text']

data/test/test_normalize.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_normalize.rb

data/test/test_page.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_page.rb
@@ -22,14 +20,14 @@ class TestPage < MiniTest::Test
 #      [ 'ls', 9  ],
 #      [ 'vt', 8  ],
       ]
-    
+
     pages.each do |rec|
         code       = rec[0]
         sects_size = rec[1]
 
         html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
         page = Factbook::Page.new( code, html: html )
-       
+
         assert_equal sects_size, page.sects.size
     end
   end

data/test/test_sanitizer.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_sanitizer.rb
@@ -18,10 +16,9 @@ class TestSanitizer < MiniTest::Test
     ## ['au'].each do |cnty|
     ['au','ag','be'].each do |cnty|
 
-      ## use/fix: ASCII-8BIT (e.g.keep as is) -???
-      html_ascii = File.read( "#{Factbook.root}/test/data/src/#{cnty}.html" )     ## fix/todo: use ASCII8BIT/binary reader ??
+      html_original = File.read( "#{Factbook.root}/test/data/src/#{cnty}.html", 'r:utf-8' ) { |f| r.read }
 
-      html, info, errors = Factbook::Sanitizer.new.sanitize( html_ascii )
+      html, info, errors = Factbook::Sanitizer.new.sanitize( html_original )
 
       File.open( "./tmp/#{cnty}.profile.html", 'w' ) do |f|
         f.write "** info:\n"

data/test/test_sanitizer_regex.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ###
 #  to run use
 #     ruby -I ./lib -I ./test test/test_sanitizer_regex.rb

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: factbook-readers
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-26 00:00:00.000000000 Z
+date: 2020-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
@@ -114,13 +114,11 @@ files:
 - Manifest.txt
 - README.md
 - Rakefile
-- data/attributes.yml
 - data/categories.csv
 - data/codes.csv
 - data/codesxref.csv
 - data/comparisons.csv
 - lib/factbook-readers.rb
-- lib/factbook-readers/attributes.rb
 - lib/factbook-readers/builder.rb
 - lib/factbook-readers/builder_item.rb
 - lib/factbook-readers/builder_json.rb
@@ -139,26 +137,12 @@ files:
 - lib/factbook-readers/utils_info.rb
 - lib/factbook-readers/version.rb
 - lib/factbook/readers.rb
-- test/data/au.html
-- test/data/au.yml
-- test/data/be.html
-- test/data/be.yml
-- test/data/json/au.json
-- test/data/src/ag.html
-- test/data/src/au-2015-09-24.html
-- test/data/src/au.html
-- test/data/src/be-2015-09-24.html
-- test/data/src/be.html
 - test/helper.rb
-- test/test_attribs.rb
-- test/test_attribs_def.rb
 - test/test_builder.rb
 - test/test_codes.rb
 - test/test_comparisons.rb
-- test/test_convert.rb
 - test/test_counter.rb
 - test/test_fields.rb
-- test/test_importer.rb
 - test/test_item_builder.rb
 - test/test_json.rb
 - test/test_json_builder.rb