factbook-readers 0.0.1

data/test/test_json.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_json.rb
+
+
+require 'helper'
+
+
+class TestJson < MiniTest::Test
+
+
+  def test_json
+     Dir.mkdir( 'tmp' )  unless Dir.exists?( 'tmp' )
+
+     codes = [ 'au',
+               'ag',
+               'be',
+               #'br',
+               #'mx',
+               #'ls',
+               #'vt',
+               #'ee',
+               #'xx'
+              ]
+
+     codes.each do |code|
+
+       html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
+       page = Factbook::Page.new( code, html: html )
+
+       h = page.data
+       pp h
+
+       ### save to json
+       puts "saving a copy to #{code}.json for debugging"
+       File.open( "tmp/#{code}.json", 'w' ) do |f|
+         f.write JSON.pretty_generate( h )
+         ## f.write page.to_json
+      end
+     end
+  end
+
+end # class TestJson

data/test/test_json_builder.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_json_builder.rb
+
+
+require 'helper'
+
+
+class TestJsonBuilder < MiniTest::Test
+
+  def test_read
+    code = 'au'
+    b = Factbook::JsonBuilder.from_file( "#{Factbook.root}/test/data/json/#{code}.json" )
+    
+    assert_equal 10, b.sects.size
+    assert_equal  1, b.sects[0].subsects.size   ## e.g. Introduction/Background
+    assert_equal 'Central Europe, north of Italy and Slovenia', b.json['Geography']['Location']['text']
+
+    assert true    ## assume everthing ok
+  end
+
+end # class TestJsonBuilder
+

data/test/test_normalize.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_normalize.rb
+
+
+require 'helper'
+
+
+class TestNormalizer < MiniTest::Test
+
+  include Factbook::NormalizeHelper
+
+  def test_normalize
+    assert_equal 'border countries', normalize_category( 'border countries:'  )
+    assert_equal 'border countries', normalize_category( 'border countries: ' )
+    assert_equal 'border countries', normalize_category( 'border countries (8):' )
+    assert_equal 'border countries', normalize_category( 'border countries (10): ' )
+  end
+
+end # class TestNormalizer
+

data/test/test_page.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_page.rb
+
+
+require 'helper'
+
+
+class TestPage < MiniTest::Test
+
+
+  def test_sects
+    pages = [
+      [ 'au', 10 ],
+      [ 'be', 10 ],
+#      [ 'br', 10 ],
+#      [ 'ee', 10 ],
+#      [ 'mx', 10 ],
+#      [ 'xx', 10 ],
+#      [ 'ls', 9  ],
+#      [ 'vt', 8  ],
+      ]
+    
+    pages.each do |rec|
+        code       = rec[0]
+        sects_size = rec[1]
+
+        html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
+        page = Factbook::Page.new( code, html: html )
+       
+        assert_equal sects_size, page.sects.size
+    end
+  end
+
+
+end # class TestPage

data/test/test_sanitizer.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_sanitizer.rb
+
+
+require 'helper'
+
+
+class TestSanitizer < MiniTest::Test
+
+  def test_sanitize
+
+    ## austria (au)
+    ## algeria (ag)
+    ## belgium (be)
+    ## ['au'].each do |cnty|
+    ['au','ag','be'].each do |cnty|
+
+      ## use/fix: ASCII-8BIT (e.g.keep as is) -???
+      html_ascii = File.read( "#{Factbook.root}/test/data/src/#{cnty}.html" )     ## fix/todo: use ASCII8BIT/binary reader ??
+
+      html, info, errors = Factbook::Sanitizer.new.sanitize( html_ascii )
+
+      File.open( "./tmp/#{cnty}.profile.html", 'w' ) do |f|
+        f.write "** info:\n"
+        f.write info.inspect + "\n\n"
+        f.write "** errors:\n"
+        f.write errors.inspect + "\n\n"
+        f.write "** html:\n"
+        f.write html
+      end
+    end
+
+    assert true    ## assume everthing ok
+  end
+
+end # class TestSanitizer

data/test/test_sanitizer_regex.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_sanitizer_regex.rb
+
+
+require 'helper'
+
+
+class TestSanitizerRegex < MiniTest::Test
+
+  def test_area_map
+
+html =<<HTML
+<div class='disTable areaComp'>
+<span class='category tCell' style='margin-bottom:0px; vertical-align:bottom;'>Area comparison map:</span>
+<span class="tCell"><a data-toggle="modal" href="#areaCompModal"><img src="../graphics/areacomparison_icon.jpg" border="0" style="cursor:pointer; border: 0px solid #CCC;"></a></span></div>
+
+        <div class="modal fade" id="areaCompModal" role="dialog">
+             <div class="wfb-modal-dialog">
+                <div class="modal-content"  >
+                   <div class="wfb-modal-header" style="border-radius: 4px; font-family: Verdana,Arial,sans-serif; font-size: 14px !important; font-weight: bold;  padding: 0.4em 16px 0.4em 1em; background: #cccccc url("..images/ui-bg_highlight-soft_75_cccccc_1x100.png") repeat-x scroll 50% 50%;" >
+                      <span style="font-size: 14px !important; margin: 0.1em 16px 0.1em 0;" class="modal-title wfb-title">The World Factbook</span><span style="float: right; margin-top: -4px;">
+                      <button type="button" class="close" title="close" data-dismiss="modal">&times;</button></span>
+                    </div>
+                   <div class="wfb-modal-body">
+...
+<div id='field'
+HTML
+
+   m = Factbook::Sanitizer::AREA_COMP_CATEGORY_REGEX.match( html )
+   pp m
+
+   assert m.nil? == false
+  end
+
+
+  def test_pop_pyramid
+
+html =<<HTML
+<div class='disTable popPyramid'>
+<span class='category tCell' style='margin-bottom:0px; vertical-align:bottom;'>population pyramid:</span>
+<span class="tCell"><a data-toggle="modal" href="#popPyramidModal"><img title="" src="../graphics/poppyramid_icon.jpg" style="cursor:pointer; border: 0px solid #CCC;"></span></a></div>
+
+        <div class="modal fade" id="popPyramidModal" role="dialog">
+             <div class="wfb-modal-dialog">
+                <div class="modal-content"  >
+                   <div class="wfb-modal-header" style="border-radius: 4px; font-family: Verdana,Arial,sans-serif; font-size: 14px !important; font-weight: bold;  padding: 0.4em 16px 0.4em 1em; background: #cccccc url("..images/ui-bg_highlight-soft_75_cccccc_1x100.png") repeat-x scroll 50% 50%;" >
+                      <span style="font-size: 14px !important; margin: 0.1em 16px 0.1em 0;" class="modal-title wfb-title">The World Factbook</span><span style="float: right; margin-top: -4px;">
+                      <button type="button" class="close" title="close" data-dismiss="modal">&times;</button></span>
+                    </div>
+                   <div class="wfb-modal-body">
+...
+<div id='field'
+HTML
+
+   m = Factbook::Sanitizer::POP_PYRAMID_CATEGORY_REGEX.match( html )
+   pp m
+
+   assert m.nil? == false
+ end  # method test_pop_pyramid
+
+
+  def test_rel_affiliation
+
+html =<<HTML
+<div class='disTable relAffiliation'><span class='category tCell' style='margin-bottom:0px; vertical-align:bottom;'>religious affiliation:</span>
+<span class="tCell"><a data-toggle="modal" href="#relAffiliationModal"><img title="" src="../graphics/middle-east-religion-icon.jpg" style="cursor:pointer; border: 0px solid #CCC;"></span></a></div>
+
+        <div class="modal fade" id="relAffiliationModal" role="dialog">
+           <div class="wfb-modal-dialog">
+              <div class="modal-content"  >
+                 <div class="wfb-modal-header" style="border-radius: 4px; font-family: Verdana,Arial,sans-serif; font-size: 14px !important; font-weight: bold;  padding: 0.4em 16px 0.4em 1em; background: #cccccc url("..images/ui-bg_highlight-soft_75_cccccc_1x100.png") repeat-x scroll 50% 50%;" >
+                    <span style="font-size: 14px !important; margin: 0.1em 16px 0.1em 0;" class="modal-title wfb-title">The World Factbook</span><span style="float: right; margin-top: -4px;">
+                    <button type="button" class="close" title="close" data-dismiss="modal">&times;</button></span>
+                 </div>
+                 <div class="wfb-modal-body">
+...
+<div id='field'
+HTML
+
+     m = Factbook::Sanitizer::REL_AFFILIATION_CATEGORY_REGEX.match( html )
+     pp m
+
+     assert m.nil? == false
+  end # method test_rel_affiliation
+
+end # class TestSanitizerRegex

metadata

@@ -0,0 +1,196 @@
+--- !ruby/object:Gem::Specification
+name: factbook-readers
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Gerald Bauer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: csvreader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webget
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.22'
+description: factbook-readers - turn thee world factbook country profile pages into
+  open structured data e.g JSON
+email: openmundi@googlegroups.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+- Rakefile
+- data/attributes.yml
+- data/categories.csv
+- data/codes.csv
+- data/codesxref.csv
+- data/comparisons.csv
+- lib/factbook-readers.rb
+- lib/factbook-readers/attributes.rb
+- lib/factbook-readers/builder.rb
+- lib/factbook-readers/builder_item.rb
+- lib/factbook-readers/builder_json.rb
+- lib/factbook-readers/codes.rb
+- lib/factbook-readers/comparisons.rb
+- lib/factbook-readers/counter.rb
+- lib/factbook-readers/normalize.rb
+- lib/factbook-readers/page.rb
+- lib/factbook-readers/page_info.rb
+- lib/factbook-readers/reader_json.rb
+- lib/factbook-readers/sanitizer.rb
+- lib/factbook-readers/sect.rb
+- lib/factbook-readers/subsect.rb
+- lib/factbook-readers/table.rb
+- lib/factbook-readers/utils.rb
+- lib/factbook-readers/utils_info.rb
+- lib/factbook-readers/version.rb
+- lib/factbook/readers.rb
+- test/data/au.html
+- test/data/au.yml
+- test/data/be.html
+- test/data/be.yml
+- test/data/json/au.json
+- test/data/src/ag.html
+- test/data/src/au-2015-09-24.html
+- test/data/src/au.html
+- test/data/src/be-2015-09-24.html
+- test/data/src/be.html
+- test/helper.rb
+- test/test_attribs.rb
+- test/test_attribs_def.rb
+- test/test_builder.rb
+- test/test_codes.rb
+- test/test_comparisons.rb
+- test/test_convert.rb
+- test/test_counter.rb
+- test/test_fields.rb
+- test/test_importer.rb
+- test/test_item_builder.rb
+- test/test_json.rb
+- test/test_json_builder.rb
+- test/test_normalize.rb
+- test/test_page.rb
+- test/test_sanitizer.rb
+- test/test_sanitizer_regex.rb
+homepage: https://github.com/factbook/factbook
+licenses:
+- Public Domain
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: factbook-readers - turn thee world factbook country profile pages into open
+  structured data e.g JSON
+test_files: []