factbook 1.1.1 → 2.0.1

data/lib/factbook/page.rb

@@ -1,185 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-
-## note:
-##   some factbook pages with chrome (headers, footers, etc.)
-##     are NOT valid utf-8, thus,
-##     treat page as is (e.g. ASCII8BIT)
-#
-#   only convert to utf8 when header and footer got stripped
-
-##
-## be/benin:
-##   Key Force or FC [Lazare S?xx?HOU?xx?TO]     -- two invalid byte code chars in Political parties and leaders:
-#
-##   in Western/Windows-1252  leads to  FC [Lazare SÈHOUÉTO];
-#       Lazare Sèhouéto
-#
-#   looks good - use (assume) Windows-1252 ????
-
-##
-#   check for is ascii 7-bit ???  if yes -noworries
-#     if not, log number of chars not using ascii 7-bit
-
-
-
-class Page
-  include LogUtils::Logging
-
-  attr_reader :sects    ## "structured" access e.g. sects/subsects/etc.
-  attr_reader :info     ##  meta info e.g. country_code, country_name, region_name, last_updated, etc.
-  attr_reader :data     ## "plain" access with vanilla hash
-
-
-  ## standard version  (note: requires https)
-  SITE_BASE = 'https://www.cia.gov/library/publications/the-world-factbook/geos/{code}.html'
-
-  def initialize( code, opts={} )
-    ### keep code - why? why not??  (use page_info/info e.g. info.country_code??)
-    
-    if opts[:json]
-      json = opts[:json]    ## note: json is (still) a string/text (NOT yet parsed to structured data)
-      b = JsonBuilder.from_string( json ) 
-    else  ## assume html
-      if opts[:html]    ## note: expects ASCII-7BIT/BINARY encoding
-         ## for debugging and testing allow "custom" passed-in html page
-        html = opts[:html]
-      else
-        url_string =  SITE_BASE.gsub( '{code}', code )
-        ## note: expects ASCII-7BIT/BINARY encoding
-        html = fetch_page( url_string )   ## use PageFetcher class - why?? why not??
-      end    
-      b = Builder.from_string( html )
-    end
-    
-    @sects = b.sects
-    @info  = b.info
-
-    ## todo/fix/quick hack:
-    ##  check for info opts hash entry - lets you overwrite page info
-    ##  -- use proper header to setup page info - why, why not??
-    if opts[:info]
-      info  = opts[:info]
-      @info = info
-    end
-
-    @data = {}    
-    @sects.each do |sect|
-      @data[ sect.title ] = sect.data
-    end
-
-    self  ## return self (check - not needed??)
-  end
-
-
-  def to_json( opts={} )  ## convenience helper for data.to_json; note: pretty print by default!
-    if opts[:minify]
-      data.to_json
-    else
-      ## was: -- opts[:pretty] || opts[:pp] 
-      JSON.pretty_generate( data )   ## note: pretty print by default!
-    end
-  end
-
-
-  def [](key)  ### convenience shortcut
-    # lets you use
-    #   page['geo']
-    #   instead of
-    #   page.data['geo']
-
-    ##  fix: use delegate data, [] from forwardable lib - why?? why not??
-
-    data[key]
-  end
-
-  ## add convenience (shortcut) accessors / attributes / fields / getters
-  
-  ATTRIBUTES.each do |attrib|
-    ## e.g.
-    ##    def background()  data['Introduction']['Background']['text']; end  
-    ##    def location()    data['Geography']['Location']['text'];      end
-    ##    etc.
-    if attrib.path.size == 1
-      define_method attrib.name.to_sym do
-        @data.fetch( attrib.category, {} ).
-              fetch( attrib.path[0], {} )['text']
-      end
-    else  ## assume size 2 for now
-      define_method attrib.name.to_sym do
-        @data.fetch( attrib.category, {} ).
-              fetch( attrib.path[0], {} ).
-              fetch( attrib.path[1], {} )['text']
-      end
-    end
-  end   
-
-
-private
-  def fetch_page( url_string )
-
-    worker = Fetcher::Worker.new
-    response = worker.get_response( url_string )
-
-    if response.code == '200'
-      t = response.body
-      ###
-      # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-      # will mostly be ASCII
-       # - try to change encoding to UTF-8 ourselves
-      logger.debug "t.encoding.name (before): #{t.encoding.name}"
-      #####
-      # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-      t
-    else
-      logger.error "fetch HTTP - #{response.code} #{response.message}"
-      ## todo/fix: raise http exception (see fetcher)  -- why? why not??
-      fail "fetch HTTP - #{response.code} #{response.message}"
-      nil
-    end
-  end
-
-
-=begin
-def self.from_url( cc, cn )
-  html_ascii = PageFetcher.new.fetch( cc )
-  self.new( cc, cn, html_ascii )
-end
-
-def self.from_file( cc, cn, opts={} )
-  input_dir = opts[:input_dir] || '.'
-  html_ascii = File.read( "#{input_dir}/#{cc}.html" )    ## fix/todo: use ASCII8BIT/binary reader
-  self.new( cc, cn, html_ascii )
-end
-=end
-
-
-end # class Page
-
-
-=begin
-class PageFetcher
-
-def fetch( cc )
-  worker = Fetcher::Worker.new
-  factbook_base = 'https://www.cia.gov/library/publications/the-world-factbook/geos'
-
-  res = worker.get_response( "#{factbook_base}/#{cc}.html" )
-
-  # on error throw exception - why? why not??
-  if res.code != '200'
-    raise Fetcher::HttpError.new( res.code, res.message )
-  end
-
-  ###
-  # Note: Net::HTTP will NOT set encoding UTF-8 etc.
-  #   will be set to ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-  html = res.body.to_s
-end
-end # PageFetcher
-=end
-
-
-end # module Factbook

data/lib/factbook/page_info.rb

@@ -1,12 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-PageInfo = Struct.new( :country_code,
-                       :country_name,
-                       :country_affiliation,
-                       :region_code,
-                       :region_name,
-                       :last_updated )
-
-end # module Factbook

data/lib/factbook/reader_json.rb

@@ -1,51 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-
-class JsonPageReader
-  def initialize( json_dir )
-    @json_dir = json_dir
-  end
-
-def read_page( code )
-  path = "#{@json_dir}/#{region_to_slug(code.region)}/#{code.code}.json"
-
-  puts "reading #{code.code} #{code.name} (#{code.region}) [#{path}]..."
-  json = File.read( path )
-
-## todo/fix/quick hack: for now until we have a proper header/meta/info section in json
-#    add some page info from code struct
-
-  info = PageInfo.new
-  info.country_code = code.code
-  info.country_name = code.name
-  info.region_name  = code.region
-  
-  page = Page.new( code.code, json: json, info: info )
-  page
-end
-
-def read_pages( codes, limit: nil )
-  pages = []
-  i=0
-  codes.each do |code|
-    next if limit && i > limit   ## for debugging just process first x entries
-    
-    pages << read_page( code )  
-  end
-  pages
-end
-
-private
-def region_to_slug( text )
-  ##  change and  =>  n
-  ##  change  &   =>  n
-  ##  change all spaces to => -
-  ##   e.g. East & Southeast Asia          => east-n-southeast-asia
-  ##        Central America and Caribbean  => central-america-n-caribbean
-  text.downcase.gsub('and', 'n').gsub( '&', 'n' ).gsub( ' ', '-' )
-end
-end ## JsonPageReader
-
-end # module Factbook

data/lib/factbook/sanitizer.rb

@@ -1,207 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-class Sanitizer
-  include LogUtils::Logging
-  include Utils     ## pulls in encode_utf8, ...
-
-
-def sanitize( html_ascii )
-  ## todo: add option for (html source) encoding - why?? why not??
-    
-  ## note:
-  ##   returns 1) html profile withouth headers, footers, scripts,etc.
-  ##           2) page (meta) info e.g. country_name, country_code, last_updated, etc.
-  ##           3) errors e.g. list of errors e.g. endcoding errors (invalid byte sequence etc.)
-
-  page_info = PageInfo.new
-    
-  h = find_page_info( html_ascii )
-  page_info.country_code        = h[:country_code]
-  page_info.country_name        = h[:country_name]
-  page_info.country_affiliation = h[:country_affiliation]
-  page_info.region_code         = h[:region_code]
-  page_info.region_name         = h[:region_name]
-
-  page_info.last_updated        = find_page_last_updated( html_ascii )
-
-
-  html_profile_ascii = find_country_profile( html_ascii )    ## cut-off headers, footers, scripts, etc.
-  
-  ## todo/fix: assume windows 12xx encoding!!!! for factbook - try 
-  html, errors = encode_utf8( html_profile_ascii )  ## change encoding to utf-8  (from binary/ascii8bit)
-
-  html = sanitize_profile( html )
-  
-  [html, page_info, errors]
-end
-
-
-
-BEGIN_FACTS_REGEX = /<ul\s+
-                       class="expandcollapse">
-                    /xim    ## ignore case; multi-line
-
-END_FACTS_REGEX = /<\/li>\s*
-                   <\/ul>\s*
-                   <\/tbody>\s*
-                   <\/table>
-                  /xim      ## ignore case; multi-line
-
-
-def find_country_profile( html )
-  ####
-  ## remove header (everything before)
-  ##   <ul class="expandcollapse">
-
-  pos = html.index( BEGIN_FACTS_REGEX )
-  fail "*** no begin facts marker found for page"  if pos.nil?
-
-  puts "  bingo - found BEGIN_FACTS on pos #{pos}"
-  html = html[pos..-1]
-
-  pp html[0..100]
-
-  ###
-  ## remove footer
-  ##  assume everthings after (last list item in unorder list inside a table body)
-  ##    </li>
-  ##    </ul>
-  ##    </tbody></table>
-
-  pos = html.index( END_FACTS_REGEX )
-  fail "*** no end facts marker found for page"  if pos.nil?
-    
-  puts "  bingo - found END_FACTS on pos #{pos}"
-  html = html[0...pos] + "</li></ul>\n"        ## note: use ... (not .. to cut-off pos)
-
-  pp html[-200..-1]
-  html
-end
-
-
-
-STYLE_ATTR_REGEX = /\s*
-                     style=('|").+?\1     ## note: use non-greedy match e.g. .+?
-                   /xim    ## do NOT allow multi-line - why? why not?
-
-CLASS_ATTR_REGEX =  /\s*
-                     class=('|")(.+?)\1     ## note: use non-greedy match e.g. .+?
-                   /xim    ## do NOT allow multi-line - why? why not?
-
-##
-## <div>
-##    <span class='category'>country comparison to the world:  </span>
-##    <span class='category_data'>[[191]]</span>
-## </div>
-##
-##  <span class='category'>country comparison to the world:  </span>
-##  <span class='category_data'><a href='../rankorder/2147rank.html#au'>114</a></span>
-
-
-## todo: add enclosing div too!!!
-
-COUNTRY_COMPARISON_REGEX = /
-        <div>
-         <span \s class='category'[^>]*>
-           country \s comparison \s to \s the \s world: \s*
-         <\/span>
-          \s*
-         <span \s class='category_data'[^>]*>
-          \s*
-            <a \s [^>]+>
-             .+?
-            <\/a>
-          \s*
-         <\/span>
-         <\/div>
-        /xim
-
-##
-##  <div class='wrap'>
-##     <div class='audio-player'>
-##    <audio id='audio-player-1' class='my-audio-player' src='../anthems/AU.mp3' type='audio/mp3' controls='controls'>
-##    </audio>
-##  </div></div>
-
-
-AUDIO_PLAYER_REGEX = /
-        <div \s class='wrap'>
-        <div \s class='audio-player'>
-          <audio \s [^>]+>
-          <\/audio>
-        <\/div>
-        <\/div>
-         /xim
-
-def sanitize_profile( html )
-
-  html = html.gsub( STYLE_ATTR_REGEX ) do |m|
-          puts "remove style attr:"
-          puts "#{m}"
-          ''
-        end
-
-  html = html.gsub( AUDIO_PLAYER_REGEX ) do |m|
-          puts "remove audio player:"
-          puts "#{m}"
-          ''
-        end
-
-
-  html = html.gsub( COUNTRY_COMPARISON_REGEX ) do |m|
-          puts "remove country comparison:"
-          puts "#{m}"
-          ''
-        end
-
-  ## remove/cleanup anchors (a href)
-  html = html.gsub( /<a\s+href[^>]*>(.+?)<\/a>/im ) do |_|   ## note: use .+? non-greedy match
-    puts " replace anchor (a) href >#{$1}<"
-    
-    inner_text = $1.dup ## keep a copy
-    if inner_text =~ /<img/    ## if includes image remove
-      puts "  remove image in anchor"
-      ''
-    else    ## keep inner text
-      inner_text
-    end
-  end
-
-
-  ## remove all list e.g. ul/li
-  html = html.gsub( /<\/?(li|ul)[^>]*>/im ) do |m|
-    puts " remove list >#{m}<"
-    ''
-  end
-
-  ## clean-up class attrib e.g. remove unknown classes
-  html = html.gsub( CLASS_ATTR_REGEX ) do |m|
-          puts "cleanup class attr:"
-          puts "#{m}"
-          
-          klasses = $2.split(' ')
-          klasses = klasses.select do |klass|
-            if ['region', 'category', 'category_data'].include?( klass )
-              true
-            else
-              puts "  remove class #{klass}"
-              false
-            end
-          end
-
-          if klasses.size > 0
-            " class='#{klasses.join(' ')}'"   ## note: add leading space!!
-          else
-            ''   ## remove class attrib completely
-          end
-        end
-
-   html
-end
-
-
-end # class Sanitizer
-
-end # module Factbook