factbook-readers 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2bc67eb2f60367d8d0ef00ca718c7d8b81b4a9c8
-  data.tar.gz: f61389d6a073db31e79766c2711bbabb89b27699
+SHA256:
+  metadata.gz: e5d0dc182771764b690661c69fdc905c4197b0a47e22f8a06903087719a3bb11
+  data.tar.gz: 7e995f86a4a1a9bb307914caff0ebb2ce39b0d2fafb62e8b1d4790608d7d6c5c
 SHA512:
-  metadata.gz: 3a565e36afae190e18154bc366bbd3d1a77f06e0c51017ba34893448fd2588fa57e2c93647ef3de5338b423e63787ef248a56c512b67504a522122bb4b24e0ff
-  data.tar.gz: 1cd6b487cb5fb2a2c5b659d2dacf0481ff5368f2e85f977c145bcf46e94f16d0543bbb59dd61995fb1b137c3bb4654308119c89b6ffa7883b0023684101b17dc
+  metadata.gz: b640ed837c55588df3f27381d86aeec14677d54d47236cf486ced33952a2619341fb396d79e280a2aa93c01600151a6ea3b0baa6a6d66c7c5ad6115479a72bbd
+  data.tar.gz: 147439c2077725912bf0ede44bb30376112abc1ec7810f9f17d46643cbc055e46ff0e2051e07098a284ff4c1990edd0051246cbd04eebd05b0603a72f1f1d128

data/Manifest.txt

@@ -2,39 +2,17 @@ CHANGELOG.md
 Manifest.txt
 README.md
 Rakefile
-data/categories.csv
-data/codes.csv
-data/codesxref.csv
-data/comparisons.csv
 lib/factbook-readers.rb
-lib/factbook-readers/builder.rb
-lib/factbook-readers/builder_item.rb
-lib/factbook-readers/builder_json.rb
-lib/factbook-readers/codes.rb
-lib/factbook-readers/comparisons.rb
+lib/factbook-readers/convert.rb
 lib/factbook-readers/counter.rb
-lib/factbook-readers/normalize.rb
 lib/factbook-readers/page.rb
 lib/factbook-readers/page_info.rb
-lib/factbook-readers/reader_json.rb
-lib/factbook-readers/sanitizer.rb
-lib/factbook-readers/sect.rb
-lib/factbook-readers/subsect.rb
-lib/factbook-readers/table.rb
-lib/factbook-readers/utils.rb
-lib/factbook-readers/utils_info.rb
 lib/factbook-readers/version.rb
 lib/factbook/readers.rb
 test/helper.rb
-test/test_builder.rb
-test/test_codes.rb
-test/test_comparisons.rb
 test/test_counter.rb
+test/test_download.rb
 test/test_fields.rb
-test/test_item_builder.rb
 test/test_json.rb
-test/test_json_builder.rb
-test/test_normalize.rb
 test/test_page.rb
-test/test_sanitizer.rb
-test/test_sanitizer_regex.rb
+test/test_version.rb

data/README.md

@@ -7,6 +7,11 @@
 * forum :: [groups.google.com/group/openmundi](https://groups.google.com/group/openmundi)
 
 
+## What's the World Factbook?
+
+See [factbook/factbook.json »](https://github.com/factbook/factbook.json)
+
+
 
 ## Usage
 
@@ -14,7 +19,7 @@
 
 ```ruby
 page = Factbook::Page.new( 'br' )   # br is the country code for Brazil
-pp page.data                        # pretty print hash
+pp page.to_h                        # pretty print data hash
 ```
 
 resulting in:
@@ -94,78 +99,15 @@ end
 ```
 
 
-### List all codes
-
-```ruby
-Factbook.codes.each do |code|
-  pp code
-end
-```
-
-resulting in:
-
-```
-#<struct Factbook::Codes::Code
- code    ="af",
- name    ="Afghanistan",
- category="Countries",
- region  ="South Asia">
-#<struct Factbook::Codes::Code
- code    ="al",
- name    ="Albania",
- category="Countries",
- region  ="Europe">
-#<struct Factbook::Codes::Code
- code    ="ag",
- name    ="Algeria",
- category="Countries",
- region  ="Africa">
-#<struct Factbook::Codes::Code
- code    ="an",
- name    ="Andorra",
- category="Countries",
- region  ="Europe">
-...
-```
-
-Note: You can filter codes by category e.g. Countries, Dependencies, Miscellaneous, Oceans, etc.
-and/or by region e.g. Africa, Europe, South Asia, Central America and Caribbean, etc.
 
 
-```ruby
-
-assert_equal 261, Factbook.codes.size
-
-## categories
-assert_equal 195, Factbook.codes.countries.size
-assert_equal  52, Factbook.codes.dependencies.size
-assert_equal   5, Factbook.codes.oceans.size
-assert_equal   1, Factbook.codes.world.size
-assert_equal   2, Factbook.codes.others.size
-assert_equal   6, Factbook.codes.misc.size
-
-## regions
-assert_equal  55, Factbook.codes.europe.size
-assert_equal   9, Factbook.codes.south_asia.size
-assert_equal   6, Factbook.codes.central_asia.size
-assert_equal  22, Factbook.codes.east_n_souteast_asia.size
-assert_equal  19, Factbook.codes.middle_east.size
-assert_equal  56, Factbook.codes.africa.size
-assert_equal   7, Factbook.codes.north_america.size
-assert_equal  33, Factbook.codes.central_america_n_caribbean.size
-assert_equal  14, Factbook.codes.south_america.size
-assert_equal  30, Factbook.codes.australia_oceania.size
-assert_equal   4, Factbook.codes.antartica.size
-assert_equal   5, Factbook.codes.region('Oceans').size
-assert_equal   1, Factbook.codes.region('World').size
-
-## categories + regions
-assert_equal  45, Factbook.codes.countries.europe.size
-...
-```
 
-See [`data/codes.csv`](data/codes.csv) for the built-in listing of all codes with categories and regions.
+## Ready-To-Use Public Domain (Free) Factbook Datasets
 
+[factbook/factbook.json](https://github.com/factbook/factbook.json) - open (public domain)
+factbook country profiles in JSON for all the world's countries (note: using the original
+/ official two-letter GEC (formerly FIPS) codes and NOT the ISO codes - you might be used to for country codes e.g. Austria is `au.json` and NOT `at.json`,
+Germany is `gm.json` and NOT `de.json` so on)
 
 
 

data/Rakefile

@@ -18,10 +18,10 @@ Hoe.spec 'factbook-readers' do
   self.history_file = 'CHANGELOG.md'
 
   self.extra_deps = [
-    ['logutils' ],
-    ['csvreader'],
+    ['factbook-codes' ],
+    ['factbook-fields' ],
     ['webget'],
-    ['nokogiri'],
+    ## ['nokogiri'],
     ## ['activerecord']  # note: will include activesupport,etc.
   ]
 

data/lib/factbook-readers.rb

@@ -1,13 +1,8 @@
-## 3rd party gems/libs
-## require 'props'
+require 'factbook-fields'
 
-require 'logutils'
+## more 3rd party gems/libs
+## require 'props'
 require 'webget'
-require 'csvreader'
-
-
-require 'nokogiri'
-
 
 
 
@@ -15,40 +10,10 @@ require 'nokogiri'
 require 'factbook-readers/version' # let it always go first
 
 
-require 'factbook-readers/codes'
-require 'factbook-readers/comparisons'
-
-
-## note: make codes, comparisons available
-module Factbook
-  ##  note: load on demand only builtin codes, comparisons, etc.
-  ##          for now
-  def self.codes
-    @@codes       ||= Codes.read_csv( "#{Factbook::Module::Readers.root}/data/codes.csv" );
-  end
-  def self.comparisons
-    @@comparisons ||= Comparisons.read_csv( "#{Factbook::Module::Readers.root}/data/comparisons.csv" )
-  end
-end # module Factbook
 
-
-
-require 'factbook-readers/utils'
-require 'factbook-readers/utils_info'
-require 'factbook-readers/sanitizer'
-require 'factbook-readers/normalize'
-require 'factbook-readers/builder_item'
-require 'factbook-readers/builder'
-require 'factbook-readers/builder_json'
-require 'factbook-readers/page'
+require 'factbook-readers/convert'
 require 'factbook-readers/page_info'
-require 'factbook-readers/sect'
-require 'factbook-readers/subsect'
-
-
-require 'factbook-readers/reader_json'
-
-require 'factbook-readers/table'    ## e.g. TableReader
+require 'factbook-readers/page'
 
 require 'factbook-readers/counter'
 

data/lib/factbook-readers/convert.rb

@@ -0,0 +1,37 @@
+
+
+def convert_cia( cia )
+ ## convert from "raw" on-the-wire cia format to
+ ##   "standard" compact "classic" format
+
+  data = {}
+
+  cia['categories'].each do |cia_cat|
+     cat = data[ cia_cat['title'] ] = {}
+     cia_cat['fields'].each do |cia_field|
+       field = cat[ cia_field['name'] ] = {}
+       if cia_field['subfields']
+         cia_field['subfields'].each do |cia_subfield|
+           subfield = field[ cia_subfield['name'] ] = {}
+           subfield[ 'text' ] = cia_subfield['content']
+         end
+
+         puts "== #{cia_cat['title']} / #{cia_field['name']} - skipping field content (w/ subfields):"
+         puts "   >#{cia_field['content']}<"
+         puts "   ?? same as:"
+         cia_field['subfields'].each do |cia_subfield|
+           puts "   #{cia_subfield['name']}: >#{cia_subfield['content']}<"
+         end
+
+       else
+         field[ 'text' ] = cia_field['content']
+       end
+
+       if cia_field[ 'field_note' ]
+         field[ 'note' ] = cia_field[ 'field_note' ]
+       end
+     end
+  end
+
+  data
+end

data/lib/factbook-readers/counter.rb

@@ -9,20 +9,20 @@ def initialize
   @data = {}
 end
 
-def count( page )
+def count( code, page )
 
   ## walk page data hash
   #   add nodes to data
 
-  walk( page, page.data, @data )
+  walk( code, page.to_h, @data )
 end
 
 
 private
-def walk( page, hin, hout )
+def walk( code, hin, hout )
    hin.each do |k,v|
-     if v.is_a? Hash
-        hout2 =  hout[k] || { count: 0, codes: '' }
+     if v.is_a?( Hash )
+        hout2 =  hout[k] ||= { count: 0, codes: '' }
 
         hout2[ :count ] += 1
 
@@ -32,12 +32,10 @@ def walk( page, hin, hout )
         codes = hout2[ :codes ]
         if codes    ## note: might got deleted if passed treshhold (e.g. 9 entries)
           codes << ' '  unless codes.empty?   ## add separator (space for now)
-          codes << page.info.country_code
+          codes << code
           hout2[ :codes ] = codes
         end
-
-        hout[k] = hout2
-        walk( page, v, hout2 )
+        walk( code, v, hout2 )
      end
    end
 end

data/lib/factbook-readers/page.rb

@@ -5,23 +5,12 @@ module Factbook
 class Page
   include LogUtils::Logging
 
-  attr_reader :sects    ## "structured" access e.g. sects/subsects/etc.
-  attr_reader :info     ##  meta info e.g. country_code, country_name, region_name, last_updated, etc.
-  attr_reader :data     ## "plain" access with vanilla hash
-
+  attr_reader :info       ##  meta info e.g. country_code, country_name, region_name, updated, etc.
 
   ## standard version  (note: requires https)
-  SITE_BASE = 'https://www.cia.gov/library/publications/the-world-factbook/geos/{code}.html'
-
+  SITE_BASE = "https://www.cia.gov/the-world-factbook/geos/{code}.json"
 
-  def self.parse( html )   ## parse html from string
-    new( html: html )
-  end
 
-  def self.read( path )
-    html = File.open( path, 'r:utf-8' ) { |f| f.read }
-    new( html: html )
-  end
 
   def self.parse_json( json )  ## parse json from string
     new( json: json )
@@ -36,87 +25,78 @@ class Page
     new( code, cache: cache )
   end
 
-  ## some convenience alias(es)
-  class << self
-    alias_method :read_html,  :read
-    alias_method :parse_html, :parse
-  end
 
 
   def initialize( code=nil,
                   json: nil,
-                  html: nil,
                   cache: false,
                   info: nil )
     if json
        ## note: assumes json is (still) a string/text
        ##        (NOT yet parsed to structured data)
-      b = JsonBuilder.new( json )
-    else  ## assume html
-      if html
-        ## for debugging and testing allow "custom" passed-in html page
-      else
+      b = ProfileBuilder.new( json )
+    else  ## assume "raw" json dataset
         ## allow passing in code struct too - just use/pluck two-letter code from struct !!!
         code = code.code   if code.is_a?( Codes::Code )
 
         raise ArgumentError, "two letter code (e.g. au) required to download page & build page url"   if code.nil?
         url = SITE_BASE.sub( '{code}', code )
 
-        html = if cache && Webcache.exist?( url )
-                   Webcache.read( url )  ## for debugging - read from cache
-               else
-                   download_page( url )
-               end
-      end
-      b = Builder.new( html )
+        raw_data = if cache && Webcache.exist?( url )
+                     text = Webcache.read( url )  ## for debugging - read from cache
+                     JSON.parse( text )
+                   else
+                     download_data( url )
+                   end
+
+## meta info from raw date - example:
+##   "name": "Aruba",
+##   "code": "AA",
+##   "region": "Central America",
+##   "published": "2021-01-25 09:07:08 -0500",
+##   "updated": "2021-01-22 14:38:14 -0500",
+##
+## note: published is NOT before updated (like an alias for created) BUT is often older/later than updated - why!?
+
+        @info = PageInfo.new
+
+        @info.country_code = raw_data['code'].downcase
+        @info.country_name = raw_data['name']
+        @info.region_name  = raw_data['region']
+
+        ## note: just parse year,month,day for now (skip hours,minutes,etc.)
+        @info.published    = Date.strptime( raw_data['published'], '%Y-%m-%d' )
+        @info.updated      = Date.strptime( raw_data['updated'], '%Y-%m-%d' )
+
+        data = convert_cia( raw_data )
+        b = ProfileBuilder.new( data )
     end
 
-    @sects = b.sects
-    @info  = b.info
+    @profile = b.profile
 
     ## todo/fix/quick hack:
     ##  check for info opts - lets you overwrite page info
     ##  -- use proper header to setup page info - why, why not??
     @info = info    if info
-
-
-    @data = {}
-    @sects.each do |sect|
-      @data[ sect.title ] = sect.data
-    end
   end
 
 
-  def to_json( minify: false )  ## convenience helper for data.to_json; note: pretty print by default!
-    if minify
-      data.to_json
-    else ## note: pretty print by default!
-      JSON.pretty_generate( data )
-    end
-  end
 
-
-  def [](key)  ### convenience shortcut
-    # lets you use
-    #   page['geo']
-    #   instead of
-    #   page.data['geo']
-
-    ##  fix: use delegate data, [] from forwardable lib - why?? why not??
-
-    data[key]
-  end
+  ## convenience helpers - forward to profile
+  def [](key)                   @profile[key]; end
+  def to_h()                    @profile.to_h; end
+  def to_json( minify: false )  @profile.to_json( minify: minify ); end
+  def size()                    @profile.size; end
 
 
 private
-  def download_page( url )
-    response = Webget.page( url )
+  def download_data( url )
+    response = Webget.call( url )
 
     ## note: exit on get / fetch error - do NOT continue for now - why? why not?
     exit 1   if response.status.nok?    ## e.g.  HTTP status code != 200
 
-
-    response.text
+    response.json
   end
 end # class Page
 end # module Factbook