factbook 2.0.0 → 2.0.1

data/lib/factbook/page.rb

@@ -1,148 +0,0 @@
-
-module Factbook
-
-
-## note:
-##   some factbook pages with chrome (headers, footers, etc.)
-##     are NOT valid utf-8, thus,
-##     treat page as is (e.g. ASCII8BIT)
-#
-#   only convert to utf8 when header and footer got stripped
-
-##
-## be/benin:
-##   Key Force or FC [Lazare S?xx?HOU?xx?TO]     -- two invalid byte code chars in Political parties and leaders:
-#
-##   in Western/Windows-1252  leads to  FC [Lazare SÈHOUÉTO];
-#       Lazare Sèhouéto
-#
-#   looks good - use (assume) Windows-1252 ????
-
-##
-#   check for is ascii 7-bit ???  if yes -noworries
-#     if not, log number of chars not using ascii 7-bit
-
-
-
-class Page
-  include LogUtils::Logging
-
-  attr_reader :sects    ## "structured" access e.g. sects/subsects/etc.
-  attr_reader :info     ##  meta info e.g. country_code, country_name, region_name, last_updated, etc.
-  attr_reader :data     ## "plain" access with vanilla hash
-
-
-  ## standard version  (note: requires https)
-  SITE_BASE = 'https://www.cia.gov/library/publications/the-world-factbook/geos/{code}.html'
-
-  def initialize( code, opts={} )
-    ### keep code - why? why not??  (use page_info/info e.g. info.country_code??)
-
-    if opts[:json]
-      json = opts[:json]    ## note: json is (still) a string/text (NOT yet parsed to structured data)
-      b = JsonBuilder.from_string( json )
-    else  ## assume html
-      if opts[:html]    ## note: expects ASCII-7BIT/BINARY encoding
-         ## for debugging and testing allow "custom" passed-in html page
-        html = opts[:html]
-      else
-        url_string =  SITE_BASE.gsub( '{code}', code )
-        ## note: expects ASCII-7BIT/BINARY encoding
-
-        ## html = fetch_page( url_string )   ## use PageFetcher class - why?? why not??
-        html = Webcache.read( url_string )
-      end
-      b = Builder.from_string( html )
-    end
-
-    @sects = b.sects
-    @info  = b.info
-
-    ## todo/fix/quick hack:
-    ##  check for info opts hash entry - lets you overwrite page info
-    ##  -- use proper header to setup page info - why, why not??
-    if opts[:info]
-      info  = opts[:info]
-      @info = info
-    end
-
-    @data = {}
-    @sects.each do |sect|
-      @data[ sect.title ] = sect.data
-    end
-
-    self  ## return self (check - not needed??)
-  end
-
-
-  def to_json( opts={} )  ## convenience helper for data.to_json; note: pretty print by default!
-    if opts[:minify]
-      data.to_json
-    else
-      ## was: -- opts[:pretty] || opts[:pp]
-      JSON.pretty_generate( data )   ## note: pretty print by default!
-    end
-  end
-
-
-  def [](key)  ### convenience shortcut
-    # lets you use
-    #   page['geo']
-    #   instead of
-    #   page.data['geo']
-
-    ##  fix: use delegate data, [] from forwardable lib - why?? why not??
-
-    data[key]
-  end
-
-  ## add convenience (shortcut) accessors / attributes / fields / getters
-
-  ATTRIBUTES.each do |attrib|
-    ## e.g.
-    ##    def background()  data['Introduction']['Background']['text']; end
-    ##    def location()    data['Geography']['Location']['text'];      end
-    ##    etc.
-    if attrib.path.size == 1
-      define_method attrib.name.to_sym do
-        @data.fetch( attrib.category, {} ).
-              fetch( attrib.path[0], {} )['text']
-      end
-    else  ## assume size 2 for now
-      define_method attrib.name.to_sym do
-        @data.fetch( attrib.category, {} ).
-              fetch( attrib.path[0], {} ).
-              fetch( attrib.path[1], {} )['text']
-      end
-    end
-  end
-
-
-private
-  def fetch_page( url )
-    response = Webget.page( url )
-
-    ## note: exit on get / fetch error - do NOT continue for now - why? why not?
-    exit 1   if response.status.nok?    ## e.g.  HTTP status code != 200
-
-
-    response.text
-  end
-
-
-=begin
-def self.from_url( cc, cn )
-  html_ascii = PageFetcher.new.fetch( cc )
-  self.new( cc, cn, html_ascii )
-end
-
-def self.from_file( cc, cn, opts={} )
-  input_dir = opts[:input_dir] || '.'
-  html_ascii = File.read( "#{input_dir}/#{cc}.html" )    ## fix/todo: use ASCII8BIT/binary reader
-  self.new( cc, cn, html_ascii )
-end
-=end
-
-
-end # class Page
-end # module Factbook

data/lib/factbook/page_info.rb

@@ -1,12 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-PageInfo = Struct.new( :country_code,
-                       :country_name,
-                       :country_affiliation,
-                       :region_code,
-                       :region_name,
-                       :last_updated )
-
-end # module Factbook

data/lib/factbook/reader_json.rb

@@ -1,51 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-
-class JsonPageReader
-  def initialize( json_dir )
-    @json_dir = json_dir
-  end
-
-def read_page( code )
-  path = "#{@json_dir}/#{region_to_slug(code.region)}/#{code.code}.json"
-
-  puts "reading #{code.code} #{code.name} (#{code.region}) [#{path}]..."
-  json = File.read( path )
-
-## todo/fix/quick hack: for now until we have a proper header/meta/info section in json
-#    add some page info from code struct
-
-  info = PageInfo.new
-  info.country_code = code.code
-  info.country_name = code.name
-  info.region_name  = code.region
-  
-  page = Page.new( code.code, json: json, info: info )
-  page
-end
-
-def read_pages( codes, limit: nil )
-  pages = []
-  i=0
-  codes.each do |code|
-    next if limit && i > limit   ## for debugging just process first x entries
-    
-    pages << read_page( code )  
-  end
-  pages
-end
-
-private
-def region_to_slug( text )
-  ##  change and  =>  n
-  ##  change  &   =>  n
-  ##  change all spaces to => -
-  ##   e.g. East & Southeast Asia          => east-n-southeast-asia
-  ##        Central America and Caribbean  => central-america-n-caribbean
-  text.downcase.gsub('and', 'n').gsub( '&', 'n' ).gsub( ' ', '-' )
-end
-end ## JsonPageReader
-
-end # module Factbook

data/lib/factbook/sanitizer.rb

@@ -1,178 +0,0 @@
-
-module Factbook
-
-class Sanitizer
-  include LogUtils::Logging
-  include Utils     ## pulls in encode_utf8, ...
-
-
-def sanitize( html_ascii )
-  ## todo: add option for (html source) encoding - why?? why not??
-
-  ## note:
-  ##   returns 1) html profile withouth headers, footers, scripts,etc.
-  ##           2) page (meta) info e.g. country_name, country_code, last_updated, etc.
-  ##           3) errors e.g. list of errors e.g. endcoding errors (invalid byte sequence etc.)
-
-  page_info = PageInfo.new
-
-  ## todo:
-  ##   make page info optional? why? why not?
-  ##   not always available (if page structure changes) - check
-  ##   what page info is required??
-  h = find_page_info( html_ascii )
-  if h
-    page_info.country_code        = h[:country_code]
-    page_info.country_name        = h[:country_name]
-    page_info.country_affiliation = h[:country_affiliation]
-    page_info.region_code         = h[:region_code]
-    page_info.region_name         = h[:region_name]
-  else
-    page_info.country_code = find_country_code( html_ascii )
-    ## print/warn: no page info found
-  end
-
-
-  page_info.last_updated        = find_page_last_updated( html_ascii )
-
-
-  html = find_country_profile( html_ascii )    ## cut-off headers, footers, scripts, etc.
-
-  ## todo/fix: assume windows 12xx encoding!!!! for factbook - try
-  # html, errors = encode_utf8( html_profile_ascii )  ## change encoding to utf-8  (from binary/ascii8bit)
-
-  # html = sanitize_profile( html )
-
-  [html, page_info, []]
-end
-
-
- #
-  #  <span class="subfield-date" aria-label="Date of information: 2018">(2018)</span>
-  #
-  #  remove aria labels
-  ARIA_ATTR_REGEX = /\s*
-                       aria-label=('|").+?\1     ## note: use non-greedy match e.g. .+?
-                     /xim    ## do NOT allow multi-line - why? why not?
-
-
-def find_country_profile( html )
-  ####
-  ## remove header (everything before)
-  ##   <ul class="expandcollapse">
-
-  doc = Nokogiri::HTML( html )
-
-  ul = doc.css( 'ul.expandcollapse' )[0]
-
-  puts ul.to_html[0..100]
-
-
-
-  ## note: special case cc uses h2 instead of div block
-  ##  <h2 class="question cam_med" sectiontitle="Introduction" ccode="cc"
-  ##         style="border-bottom: 2px solid white; cursor: pointer;">
-  ##         Introduction ::  <span class="region">CURACAO </span>
-  ##   </h2>
-  ##   is old format !!!!
-  ##   cc - CURACAO
-  ##  http headers says - last-modified: Wed, 14 Nov 2018 14:09:28 GMT
-  ##   page says - PAGE LAST UPDATED ON MARCH 14, 2018
-  ##    wait for new version to be generated / pushed!!!
-
-  ## check for old format if h2 are present
-  h2s = ul.css( 'h2' )
-  if h2s.size > 0
-    puts "  !! WARN: found #{h2s.size} h2(s) - assume old format - sorry - must wait for update!!!"
-    ## return empty html string - why? why not?
-    return  ''
-  end
-
-
-  ###
-  ## sanitize
-
-  ## remove link items
-  ##   assume two <li>s are a section
-
-  html = String.new('')
-
-  ##  filter all li's
-  ul_children = ul.children.select { |el| if el.name == 'li'
-                                             true
-                                          else
-                                            # puts "skipping #{el.name} >#{el.to_html}<"
-                                            false
-                                          end
-                                    }
-  puts "  #{ul_children.size} li(s):"
-  ul_children.each_slice(2) do |lis|
-    li  = lis[0]
-    div = li.at( 'div[sectiontitle]' )
-    if div.nil?
-      puts "!! ERROR: no section title found in div:"
-      puts li.to_html
-      exit 1
-    end
-
-    section_title = div['sectiontitle'].to_s
-
-    html << "<h2>#{section_title}</h2>\n"
-
-
-    li  = lis[1]
-    ## filter all div's
-    li_children = li.children.select { |el| if el.name =='div'
-                                                true
-                                            else
-                                             # puts "skipping #{el.name} >#{el.to_html}<"
-                                             false
-                                            end
-                                      }
-    puts " #{li_children.size} div(s):"
-
-    li_children.each_slice(2) do |divs|
-      div = divs[0]
-      a = div.css('a')[0]
-
-      if a
-        html << "\n<h3>#{a.text}:</h3>\n"
-      else
-        puts "!! WARN: no anchor found:"
-        puts div.to_html
-      end
-
-
-      div = divs[1]
-      div_children = div.children.select {|el| el.name == 'div' ? true : false }
-      div_children.each do |catdiv|
-         if catdiv['class'] && catdiv['class'].index( 'category_data' )
-
-          if catdiv['class'].index( 'attachment' )
-            ## skip attachments e.g. maps, pop pyramids, etc.
-          else
-            html << catdiv.to_html
-            html << "\n"
-          end
-         else
-          puts "!! WARN: skipping div (W/O category_data class):"
-          puts catdiv.to_html
-         end
-      end
-    end
-  end
-
-
-  html = html.gsub( ARIA_ATTR_REGEX ) do |m|
-    puts "remove aria-label attr:"
-    puts "#{m}"
-    ''
-  end
-
-  html
-end
-
-
-end # class Sanitizer
-
-end # module Factbook

data/lib/factbook/sect.rb

@@ -1,29 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-
-class Sect
-  include LogUtils::Logging
-
-  attr_accessor :title        ## use name instead of title - why? why not? 
-  attr_accessor :subsects
-  
-  def initialize
-    @subsects = []
-  end
-
-  def data
-    ## convert sects to hash
-    @data = {}
-    
-    subsects.each_with_index do |subsect,i|
-      @data[ subsect.title ] = subsect.data
-    end
-    @data
-  end
-
-
-end # class Sect
-
-end # module Factbook

data/lib/factbook/subsect.rb

@@ -1,18 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-
-class Subsect
-  include LogUtils::Logging
-
-  attr_accessor :title        ## use name instead of title - why? why not?
-  attr_accessor :data         ## hash holding data e.g. { 'text' => '...' etc. }
-
-  def initialize
-    @data = {}
-  end
-
-end # class Subsect
-
-end # module Factbook

data/lib/factbook/table.rb

@@ -1,52 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-##
-## make more "generic"  - why? why not?
-##   (re)use for other files ?? move to textutils ??
-
-##
-##  for now reads in rows with values separated by at least 3+ spaces e.g.:
-##   see www.cia.gov/library/publications/the-world-factbook/rankorder/rawdata_2119.txt
-## 1      China                      1,367,485,388                                     
-## 2      India                      1,251,695,584                                     
-## 3      European Union             513,949,445                                       
-## 4      United States              321,368,864                                       
-## 5      Indonesia                  255,993,674                                       
-## 6      Brazil                     204,259,812     
-
-
-class TableReader
-  include LogUtils::Logging
-
-
-def initialize( text )
-  @text = text
-end
-
-def read
-  recs = []
-
-  line_no = 0
-  @text.each_line do |line|
-    line_no +=1
-    line = line.strip   ## remove leading and trailing whitespace
-    if line.empty?
-      puts "** skipping empty line #{line_no}"
-      next
-    end
-
-    values = line.split( /[ ]{3,}/ )    ## split three or more spaces - use just two ?? why? why not??
-    
-    ## puts line
-    ## pp values
-    recs << values
-  end
-  recs
-end
-
-
-end # class TableReader
-
-end # module Factbook