factbook-readers 1.0.1 → 1.1.0

data/lib/factbook-readers/builder_json.rb

@@ -1,68 +0,0 @@
-
-module Factbook
-
-######
-# json builder -- lets us rebuild a page from "dumped" json (instead of parsing html page)
-
-class JsonBuilder
-  include LogUtils::Logging
-  include NormalizeHelper    ##  e.g. normalize_category
-
-
-attr_reader :text,
-            :json,
-            :info,            ## not used yet -- page info incl. country_name, region_name, last_updated etc.
-            :errors,          ## not used yet -- encoding erros etc.
-            :sects
-
-
-def initialize( text )
-  @text = text
-
-  @json = JSON.parse( text )
-
-  @info   = nil   ## fix/todo: sorry - for now no page info (use header in json - why? why not??)
-  @errors = []       ## fix/todo: sorry - for now no errors possible/tracked
-
-  @sects = []
-
-  @json.each do |k1,v1|
-    sect_title    = k1
-    sect_subsects = v1
-
-    sect = Sect.new
-    sect.title = sect_title
-
-    ## get subsections
-    subsects = []
-    sect_subsects.each do |k2,v2|
-      subsect_title = k2
-      subsect_data  = v2
-
-      subsect = Subsect.new
-      subsect.title = subsect_title
-
-      #####
-      ## note: run data hash through normalize_category (again)
-      if subsect_data.is_a?( Hash )
-        new_subsect_data = {}
-        subsect_data.each do |k3,v3|
-          new_subsect_data[ normalize_category(k3) ] = v3
-        end
-        subsect_data = new_subsect_data
-      end
-
-      subsect.data  = subsect_data
-
-      subsects << subsect
-    end
-
-    sect.subsects = subsects
-    @sects << sect
-  end
-end
-
-end # class JsonBuilder
-
-
-end # module Factbook

data/lib/factbook-readers/codes.rb

@@ -1,121 +0,0 @@
-##
-# note:
-#   the factbook category/region for world is other entities (on FAQ) and oceans in page
-#    changed to world
-
-
-module Factbook
-
-class Codes
-
-  Code = Struct.new( :code,      ## todo: add notes (country affiliation) - why? why not??
-                     :name,
-                     :category,  ## e.g. Countries, Other, Oceans, World, Dependencies, etc.
-                     :region,    ## e.g. Europe, Oceans, etc.
-                    )
-
-  def self.read_csv( path )
-    ###
-    #  note:
-    #   if you use quotes - NO leading spaces allowed e.g.
-    #  use au,"Austria",... and NOT
-    #      au, "Austria", ...
-    #
-    #  for headers - NO leading spaces allowed e.g.
-    #   use  Code,Name,Category,Region,...   and NOT
-    #        Code, Name, Category, Region, ...
-
-    rows = CsvHash.read( path )
-
-    pp rows
-
-    recs = []
-    rows.each do |row|
-      pp row
-      rec = Code.new
-      rec.code     = row['Code'].strip    ## remove leading n trailing whitespaces
-      rec.name     = row['Name'].strip
-
-      ## note: for now category and region are optional
-      rec.category = row['Category'].strip   if row['Category'] && row['Category'].size > 0
-      rec.region   = row['Region'].strip     if row['Region'] && row['Region'].size > 0
-
-      pp rec
-      recs << rec
-    end
-
-    new( recs )
-  end
-
-
-  def initialize( codes )
-    @codes = codes
-  end
-
-  def size() @codes.size; end
-
-  def each( &blk ) @codes.each( &blk ); end
-  def select( &blk )
-    codes = @codes.select( &blk )
-    Codes.new( codes )   ## return (again) new Codes obj for easy-chaining - why? why not?
-  end
-
-
-  def to_a
-    @codes.collect {|code| code.code }   ## return array of codes
-  end
-
-  ##  def all()  self.to_a; end    ## note: alias for to_a - use - why? why not??
-
-  ## "pre-defined" convenience shortcuts
-  def countries()       category 'Countries';     end
-  def world()           category 'World';         end
-  def oceans()          category 'Oceans';        end
-  def misc()            category 'Miscellaneous'; end
-  def others()          category 'Other';         end
-  def dependencies()    category 'Dependencies';  end
-  def dependencies_us() category 'Dependencies (United States)'; end
-## fix/todo: add all dependencies  uk (or gb?), fr,cn,au,nz,no,dk,etc.
-
-  def europe()               region 'Europe';            end
-  def south_asia()           region 'South Asia';        end
-  def central_asia()         region 'Central Asia';      end
-  def east_n_souteast_asia() region 'East & Southeast Asia'; end
-  def middle_east()          region 'Middle East';       end
-  def africa()               region 'Africa';            end
-  def north_america()        region 'North America';     end
-  def central_america_n_caribbean() region 'Central America and Caribbean'; end
-  def south_america()        region 'South America';     end
-  def australia_oceania()    region 'Australia-Oceania'; end
-  def antartica()            region 'Antarctica';        end
-
-  ## note: regions oceans and world - same as category oceans and world
-  ##     use oceans_ii or world_ii or something ??
-  ##   use category('World')  n region('World')
-  ##   use category('Oceans') n region('Oceans')
-
-
-  def category( query )
-    ## todo/future: allow passing in of regex too (not just string)
-    ## note: e.g. Dependencies (France) needs to get escpaed to
-    ##            Dependencies \(France\)  etc.
-    filter_regex = /#{Regexp.escape(query)}/i
-    codes = @codes.select do |code|
-      code.category ? filter_regex.match( code.category ) : false   ## note: allow nil for category; will fail on search
-    end
-    Codes.new( codes )   ## return new Codes obj for easy-chaining
-  end
-
-  def region( query )
-    ## todo/future: allow passing in of regex too (not just string)
-    filter_regex = /#{Regexp.escape(query)}/i
-    codes = @codes.select do |code|
-       code.region ? filter_regex.match( code.region ) : false      ## note: allow nil for region; will fail on search
-    end
-    Codes.new( codes )   ## return new Codes obj for easy-chaining
-  end
-
-end  # class codes
-
-end # module Factbook
-

data/lib/factbook-readers/comparisons.rb

@@ -1,49 +0,0 @@
-
-module Factbook
-
-class Comparisons
-
-  Comparison = Struct.new( :num,        ### todo: use no or id or something - why? why not?
-                           :category,  ## e.g. Geography, People, Economy, etc.
-                           :name,
-                          )
-
-  def self.read_csv( path )
-
-    rows = CsvHash.read( path )
-
-    pp rows
-
-    recs = []
-    rows.each do |row|
-      pp row
-      rec = Comparison.new
-      rec.num      = row['Num'].strip.to_i    ## remove leading n trailing whitespaces
-      rec.category = row['Category'].strip
-      rec.name     = row['Name'].strip
-
-      pp rec
-      recs << rec
-    end
-
-    new( recs )
-  end
-
-  def initialize( comps )
-    @comps = comps
-  end
-
-  def size() @comps.size; end
-
-  def each
-    @comps.each {|comp| yield( comp ) }
-  end
-
-  def to_a
-    @comps.collect {|comp| comp.num }   ## return array of nums   -- return something else - why? why not?
-  end
-
-end  # class Comparison
-
-end # module Factbook
-

data/lib/factbook-readers/normalize.rb

@@ -1,42 +0,0 @@
-
-module Factbook
-  module NormalizeHelper
-
-
-def normalize_category( text )
-
-  ## note: fix typos/errors with double colons e.g. note::  (instead of note:)
-
-  text = text.strip
-  text = text.sub( /:+\z/, '' )      # remove trailing : if present -- note: allow (fix) note:: too, thus, use :+
-  text = text.strip
-
-  #######################################
-  ### special cases
-
-  ##   typos e.g ntoe => use note
-  text = 'note'                         if text == 'ntoe'
-  text = 'investment in fixed capital'  if text == 'investment if fixed capital'
-
-  ##  downcase
-  text = 'lowest point'    if text == 'Lowest point'
-  text = 'chief of state'  if text == 'Chief of state'
-
-  ##  spelling variant (use more popular one)
-  text = 'signed, but not ratified'     if text == 'signed but not ratified'
-  text = 'vectorborne diseases'         if text == 'vectorborne disease'
-  text = 'water contact disease'        if text == 'water contact diseases'
-  text = 'food or waterborne diseases'  if text == 'food or waterborne disease'
-  text = 'geographic coordinates'       if text == 'geographical coordinates'
-  text = 'note'                         if text == 'notes'
-  text = 'refugees (country of origin)' if text == 'refugees (countries of origin)'
-
-  ##    border countries (8):   -- remove (x) counter
-  text = 'border countries'   if text.start_with?( 'border countries')
-
-  text
-end
-
-
-  end # module NormalizeHelper
-end  # module Factbook

data/lib/factbook-readers/reader_json.rb

@@ -1,50 +0,0 @@
-
-module Factbook
-
-
-class JsonPageReader
-  def initialize( json_dir )
-    @json_dir = json_dir
-  end
-
-def read_page( code )
-  path = "#{@json_dir}/#{region_to_slug(code.region)}/#{code.code}.json"
-
-  puts "reading #{code.code} #{code.name} (#{code.region}) [#{path}]..."
-  json = File.read( path, 'r:utf-8' ) { |f| f.read }
-
-## todo/fix/quick hack: for now until we have a proper header/meta/info section in json
-#    add some page info from code struct
-
-  info = PageInfo.new
-  info.country_code = code.code
-  info.country_name = code.name
-  info.region_name  = code.region
-
-  page = Page.new( code.code, json: json, info: info )
-  page
-end
-
-def read_pages( codes, limit: nil )
-  pages = []
-  i=0
-  codes.each do |code|
-    next if limit && i > limit   ## for debugging just process first x entries
-
-    pages << read_page( code )
-  end
-  pages
-end
-
-private
-def region_to_slug( text )
-  ##  change and  =>  n
-  ##  change  &   =>  n
-  ##  change all spaces to => -
-  ##   e.g. East & Southeast Asia          => east-n-southeast-asia
-  ##        Central America and Caribbean  => central-america-n-caribbean
-  text.downcase.gsub('and', 'n').gsub( '&', 'n' ).gsub( ' ', '-' )
-end
-end ## JsonPageReader
-
-end # module Factbook

data/lib/factbook-readers/sanitizer.rb

@@ -1,351 +0,0 @@
-
-module Factbook
-
-class Sanitizer
-  include LogUtils::Logging
-  include Utils   ## e.g. find_page_info etc.
-
-def sanitize( html )
-  ## todo: add option for (html source) encoding - why?? why not??
-
-  ## note:
-  ##   returns 1) html profile withouth headers, footers, scripts,etc.
-  ##           2) page (meta) info e.g. country_name, country_code, last_updated, etc.
-  ##           3) errors e.g. list of errors e.g. endcoding errors (invalid byte sequence etc.)
-
-  page_info = PageInfo.new
-
-  ## todo:
-  ##   make page info optional? why? why not?
-  ##   not always available (if page structure changes) - check
-  ##   what page info is required??
-  h = find_page_info( html )
-  if h
-    page_info.country_code        = h[:country_code]
-    page_info.country_name        = h[:country_name]
-    page_info.country_affiliation = h[:country_affiliation]
-    page_info.region_code         = h[:region_code]
-    page_info.region_name         = h[:region_name]
-  else
-    page_info.country_code = find_country_code( html )
-    ## print/warn: no page info found
-  end
-
-
-  page_info.last_updated  = find_page_last_updated( html )
-
-
-  html_profile = find_country_profile( html )    ## cut-off headers, footers, scripts, etc.
-
-  ## todo/check: remove 3rd args old errors array - why? why not?
-  [html_profile, page_info, []]
-end
-
-
-
-def find_country_profile( html )
-  ####
-  ## remove header (everything before)
-  ##   <ul class="expandcollapse">
-
-  ##
-  ## fix know broken html bugs
-  ##  in co (Columbia) page (Nov/11 2020):
-  ## <div class="photogallery_captiontext">
-  ##   <p>slightly less than twice the size of Texas</p
-  ## </div>
-  ##    note: </p    => unclosed p!! change to </p>
-
-  ## note: in regex use negative looakhead e.g. (?!patttern)
-  html = html.gsub( %r{</p(?![>])} ) do |m|
-    puts "!! WARN: fixing unclosed </p => </p>"
-    puts "#{m}"
-    '</p>'
-  end
-
-
-  doc = Nokogiri::HTML( html )
-
-  ul = doc.css( 'ul.expandcollapse' )[0]
-
-  puts ul.to_html[0..100]
-
-
-  ###
-  ## sanitize
-  ## remove link items
-  ##   assume two <li>s are a section
-
-  html = String.new('')
-
-  ##  filter all li's
-   ul_children = ul.children.select { |el| if el.name == 'li'
-                                             true
-                                          else
-                                            # puts "skipping #{el.name} >#{el.to_html}<"
-                                            false
-                                          end
-                                     }
-  ## ul_children = ul.css( 'li' )
-
-  puts "  #{ul_children.size} li(s):"
-  ul_children.each_slice(2) do |lis|
-    li  = lis[0]
-    div = li.at( 'div[sectiontitle]' )
-    if div.nil?
-      puts "!! ERROR: no section title found in div:"
-      puts li.to_html
-      exit 1
-    end
-
-    section_title = div['sectiontitle'].to_s
-
-    html << "<h2>#{section_title}</h2>\n"
-
-
-    li  = lis[1]
-    ## filter all div's
-    li_children = li.children.select { |el| if el.name =='div'
-                                                true
-                                            else
-                                             # puts "skipping #{el.name} >#{el.to_html}<"
-                                             false
-                                            end
-                                      }
-    puts " #{li_children.size} div(s) in >#{section_title}<:"
-
-
-    ## check special case in world  Geographic overview:
-#    <div class="category oce_light" style="padding-left:5px;"
-#       id="field-anchor-geography-geographic-overview">
-#           Geographic overview:
-#       <span class="field-listing-link">
-#            <a href="../fields/275.html#XX">
-#              <img alt="Geographic overview field listing"
-#                   title="Geographic overview field listing"
-#                   src="../images/field_listing.gif" /></a>
-#         </span>
-#</div>
-# vs regular
-#
-# <div class="category oce_light" style="padding-left:5px;"
-#       id="field-anchor-geography-area-comparative">
-#      <span class="btn-tooltip definition" role="tooltip" aria-hidden='true'>
- #       <a aria-label="Use this link to access a description of the Area - comparative field"
- #            href="../docs/notesanddefs.html#280">
- #            Area - comparative
- #         </a>:
- #       <span class="tooltip-content">
- #           This entry provides an area comparison based on total area equivalents. Most entities are compared with the entire US or one of the 50 states based on area measurements (1990 revised) provided by the US Bureau of the Census. The smaller entities are compared with Washington, DC (178 sq km, 69 sq mi) or The Mall in Washington, DC (0.59 sq km, 0.23 sq mi, 146 acres).
- #       </span>
- #     </span>
-  #    <span class="field-listing-link">
-  #        <a href="../fields/280.html#XX"><img alt="Area - comparative field listing" title="Area - comparative field listing" src="../images/field_listing.gif" /></a>
-  #    </span>
-  # </div>
-
-    li_children.each_slice(2) do |divs|
-      div = divs[0]
-
-      ## try new way - try clean-up / rm first
-      span_tooltip_content = div.at( 'span.tooltip-content' )
-      if span_tooltip_content
-        span_tooltip_content.inner_html = ''
-        span_tooltip_content.replace( '' )  ## check for how to delete/remove - why? why not!!
-      end
-
-      span_field_listing_link = div.at( 'span.field-listing-link' )
-      if span_field_listing_link
-        span_field_listing_link.inner_html = ''
-        span_field_listing_link.replace( '' )
-      end
-
-      subsection_title = div.text.strip
-      html << "\n<h3>#{subsection_title}</h3>\n"
-
-      # a = div.css('a')[0]
-      # if a
-      #  subsection_title = a.text   ## todo/check/rename: use field_name or such - why? why not?
-      #  html << "\n<h3>#{subsection_title}:</h3>\n"
-      # else
-      #  subsection_title = '???'
-      #  puts "!! WARN: no anchor found:"
-      #  puts div.to_html
-      # end
-
-
-      div = divs[1]
-      div_children = div.children.select {|el| el.name == 'div' ? true : false }
-      puts "   #{div_children.size} div(s) in field >#{subsection_title}<:"
-
-      ## use more robust version - only get divs with category_data
-      ## div_children = div.css( 'div.category_data' )
-      ## puts "   #{div_children.size} div(s) in field >#{subsection_title}< v2:"
-
-      # if div_children.size > 14
-      #  ## us labor force has 11 divs
-      #  ## possibly an error
-      #  puts "!! ERROR - too many category_data divs found:"
-      #  puts div.to_html[0..200]
-      #  puts "\n...\n"
-      #  puts puts div.to_html[-400..-1]
-      #  exit 1
-      # end
-
-      div_children.each do |catdiv|
-         if catdiv['class'] && catdiv['class'].index( 'category_data' )
-
-          if catdiv['class'].index( 'attachment' )
-            ## skip attachments e.g. maps, pop pyramids, etc.
-          else
-            html << sanitize_data( catdiv, title: subsection_title )
-            html << "\n"
-          end
-         else
-            if catdiv.to_html.index( 'country comparison to the world' )
-              ## simplify/unlinkify country comparision
-              ## <div>
-              ##  <span class='category'>country comparison to the world:</span>
-              ##  <span class='category_data'>
-              ##    <a href="../fields/335rank.html#AU">97</a>
-              ##  </span>
-              ## </div>
-              ##  e.g. to =>
-              ##  <div>
-              ##    country comparison to the world: 97
-              ##  </div>
-              html << "<div>\n  #{squish( catdiv.text.strip )}\n</div>"
-              html << "\n"
-            else
-              puts "!! ERROR: div (W/O category_data class) in >#{subsection_title}<:"
-              puts catdiv.to_html
-              exit 1
-            end
-         end
-      end
-    end
-  end
-
-  html
-end
-
-
-#
-#  <span class="subfield-date" aria-label="Date of information: 2018">(2018)</span>
-#
-#  remove aria labels
-ARIA_ATTR_RE = /\s*
-                     aria-label=('|").+?\1     ## note: use non-greedy match e.g. .+?
-                  /xim    ## do NOT allow multi-line - why? why not?
-
-## find double breaks e.g. <br><br>
-BR_BR_RE = /(<br> \s* <br>)
-              /xim   ## do NOT allow multi-line - why? why not?
-
-
-def sanitize_data( el, title: )
-  ## todo/fix/check:
-  ##  check if more than one p(aragraph)
-  ##    get squezzed together without space inbetween?
-
-
-  ## step 0: replace all possible a(nchor) links with just inner text
-  el.css( 'a').each do |a|
-     a.replace( " #{a.text.strip} " )
-  end
-
-
-
-  inner_html = String.new('')
-
-  ## step 1 - unwrap paragraphs if present
-  ##          and convert dom/nokogiri doc/tree to html string
-  p_count = 0
-  el.children.each do |child|
-    if child.name == 'p'
-      ## puts "  [debug ] unwrap <p> no.#{p_count+1}"
-
-      p_inner_html = child.inner_html.strip  ## note: unwrap! use inner_html NOT to_html/html
-      if p_inner_html.empty?
-        ## note: skip empty paragraphs for now
-      else
-        inner_html << ' ++ '    if p_count > 0
-        inner_html << p_inner_html
-        inner_html << " \n\n "
-
-        p_count += 1
-      end
-    else
-      inner_html << child.to_html
-    end
-  end
-  ## note: keep container div!! just replace inner html!!!
-  ##  note: right strip all trailing spaces/newlines for now
-  ##        plus add back a single one for pretty printing
-
-  ## note: replace all non-breaking spaces with spaces for now
-  ##  see fr (france) in political parties section for example
-  ##  todo/check/fix:  check if we need to use unicode char!! and NOT html entity
-  inner_html = inner_html.gsub( "&nbsp;", ' ' )
-  ## Unicode Character 'NO-BREAK SPACE' (U+00A0)
-  inner_html = inner_html.gsub( "\u00A0", ' ' )  ## use unicode char
-
-
-  el.inner_html = inner_html.rstrip + "\n"
-
-  # finally - convert back to html (string)
-  html = el.to_html
-
-
-
-  html = html.gsub( ARIA_ATTR_RE ) do |m|
-    ## do not report / keep silent for now
-    ## puts "in >#{title}< remove aria-label attr:"
-    ## puts "#{m}"
-    ''
-  end
-
-  html = html.gsub( BR_BR_RE ) do |m|
-    puts "in >#{title}< squish two <br>s into one:"
-    puts "#{m}"
-    '<br>'
-  end
-
-  html = html.gsub( /<br>/i ) do |m|
-    puts "in >#{title}< replace <br> with inline (plain) text ++:"
-    puts "#{m}"
-    ' ++ '
-  end
-
-  ## cleanup/remove ++   before subfield e.g.
-  ##  of: ++   => of:    or such
-  ##
-  ##  todo/fix: add negative lookahead e.g. not another + to be more specific!!
-  html = html.gsub( %r{
-                       (?<=([a-z]:)|(:</span>))  # note: use zero-length positive lookbehind
-                          \s+
-                          \+{2}}xim ) do |m|
-     puts "in >#{title} remove ++ before <field>: marker:"
-     puts "#{m}"
-    ' '
-  end
-
-  #####
-  # "unfancy" smart quotes to ascii - why? why not?
-  # e.g.
-  #   Following Britain’s victory => Following Britain's victory
-  html = html.tr( "’", "'" )
-  #   “full floor” House vote     => "full floor" House vote
-  html = html.tr( "“”", '""' )
-
-  html
-end
-
-def squish( str )
-  str.gsub( /[ \t\n\r]{2,}/, ' ' )  ## replace multi-spaces (incl. newlines with once space)
-end
-
-
-end # class Sanitizer
-
-end # module Factbook