factbook-readers 0.0.1

data/lib/factbook-readers/builder_json.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+
+module Factbook
+
+######
+# json builder -- lets us rebuild a page from "dumped" json (instead of parsing html page)
+
+class JsonBuilder    
+  include LogUtils::Logging
+  include NormalizeHelper    ##  e.g. normalize_category
+
+
+def self.from_file( path )
+  text = File.read( path )     ## fix: use File.read_utf8  from textutils
+  self.from_string( text )
+end
+
+def self.from_string( text )
+  self.new( text )
+end
+
+
+attr_reader :text,
+            :json,
+            :info,            ## not used yet -- page info incl. country_name, region_name, last_updated etc.
+            :errors,          ## not used yet -- encoding erros etc.
+            :sects
+
+
+def initialize( text )
+  @text = text
+    
+  @json = JSON.parse( text )
+
+  @info   = nil   ## fix/todo: sorry - for now no page info (use header in json - why? why not??)
+  @errors = []       ## fix/todo: sorry - for now no errors possible/tracked
+
+  @sects = []
+
+  @json.each do |k1,v1|
+    sect_title    = k1
+    sect_subsects = v1
+    
+    sect = Sect.new
+    sect.title = sect_title
+    
+    ## get subsections
+    subsects = []
+    sect_subsects.each do |k2,v2|
+      subsect_title = k2
+      subsect_data  = v2
+      
+      subsect = Subsect.new
+      subsect.title = subsect_title
+
+      #####
+      ## note: run data hash through normalize_category (again)
+      if subsect_data.is_a?( Hash )
+        new_subsect_data = {}
+        subsect_data.each do |k3,v3|
+          new_subsect_data[ normalize_category(k3) ] = v3
+        end
+        subsect_data = new_subsect_data
+      end  
+ 
+      subsect.data  = subsect_data
+      
+      subsects << subsect
+    end
+    
+    sect.subsects = subsects
+    @sects << sect
+  end
+end
+
+end # class JsonBuilder
+
+
+end # module Factbook

data/lib/factbook-readers/codes.rb

@@ -0,0 +1,122 @@
+# encoding: utf-8
+
+##
+# note:
+#   the factbook category/region for world is other entities (on FAQ) and oceans in page
+#    changed to world
+
+
+module Factbook
+
+class Codes
+
+  Code = Struct.new( :code,      ## todo: add notes (country affiliation) - why? why not??
+                     :name,
+                     :category,  ## e.g. Countries, Other, Oceans, World, Dependencies, etc.
+                     :region,    ## e.g. Europe, Oceans, etc.
+                    )
+
+  def self.from_csv( path )
+    ###
+    #  note:
+    #   if you use quotes - NO leading spaces allowed e.g.
+    #  use au,"Austria",... and NOT
+    #      au, "Austria", ...
+    #
+    #  for headers - NO leading spaces allowed e.g.
+    #   use  Code,Name,Category,Region,...   and NOT
+    #        Code, Name, Category, Region, ...
+
+    rows = CsvHash.read( path )
+
+    pp rows
+
+    recs = []
+    rows.each do |row|
+      pp row
+      rec = Code.new
+      rec.code     = row['Code'].strip    ## remove leading n trailing whitespaces
+      rec.name     = row['Name'].strip
+
+      ## note: for now category and region are optional
+      rec.category = row['Category'].strip   if row['Category'] && row['Category'].size > 0
+      rec.region   = row['Region'].strip     if row['Region'] && row['Region'].size > 0
+
+      pp rec
+      recs << rec
+    end
+
+    self.new( recs )
+  end
+
+  def initialize( codes )
+    @codes = codes
+  end
+
+  def size() @codes.size; end
+
+  def each( &blk ) @codes.each( &blk ); end
+  def select( &blk )
+    codes = @codes.select( &blk )
+    Codes.new( codes )   ## return (again) new Codes obj for easy-chaining - why? why not?
+  end
+
+
+  def to_a
+    @codes.collect {|code| code.code }   ## return array of codes
+  end
+
+  ##  def all()  self.to_a; end    ## note: alias for to_a - use - why? why not??
+
+  ## "pre-defined" convenience shortcuts
+  def countries()       category 'Countries';     end
+  def world()           category 'World';         end
+  def oceans()          category 'Oceans';        end
+  def misc()            category 'Miscellaneous'; end
+  def others()          category 'Other';         end
+  def dependencies()    category 'Dependencies';  end
+  def dependencies_us() category 'Dependencies (United States)'; end
+## fix/todo: add all dependencies  uk (or gb?), fr,cn,au,nz,no,dk,etc.
+
+  def europe()               region 'Europe';            end
+  def south_asia()           region 'South Asia';        end
+  def central_asia()         region 'Central Asia';      end
+  def east_n_souteast_asia() region 'East & Southeast Asia'; end
+  def middle_east()          region 'Middle East';       end
+  def africa()               region 'Africa';            end
+  def north_america()        region 'North America';     end
+  def central_america_n_caribbean() region 'Central America and Caribbean'; end
+  def south_america()        region 'South America';     end
+  def australia_oceania()    region 'Australia-Oceania'; end
+  def antartica()            region 'Antarctica';        end
+
+  ## note: regions oceans and world - same as category oceans and world
+  ##     use oceans_ii or world_ii or something ??
+  ##   use category('World')  n region('World')
+  ##   use category('Oceans') n region('Oceans')
+
+
+  def category( query )
+    ## todo/future: allow passing in of regex too (not just string)
+    ## note: e.g. Dependencies (France) needs to get escpaed to
+    ##            Dependencies \(France\)  etc.
+    filter_regex = /#{Regexp.escape(query)}/i
+    codes = @codes.select do |code|
+      code.category ? filter_regex.match( code.category ) : false   ## note: allow nil for category; will fail on search
+    end
+    Codes.new( codes )   ## return new Codes obj for easy-chaining
+  end
+
+  def region( query )
+    ## todo/future: allow passing in of regex too (not just string)
+    filter_regex = /#{Regexp.escape(query)}/i
+    codes = @codes.select do |code|
+       code.region ? filter_regex.match( code.region ) : false      ## note: allow nil for region; will fail on search
+    end
+    Codes.new( codes )   ## return new Codes obj for easy-chaining
+  end
+
+end  # class codes
+
+end # module Factbook
+

data/lib/factbook-readers/comparisons.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+
+module Factbook
+
+class Comparisons
+
+  Comparison = Struct.new( :num,        ### todo: use no or id or something - why? why not?
+                           :category,  ## e.g. Geography, People, Economy, etc.
+                           :name,
+                          )
+
+  def self.from_csv( path )
+
+    rows = CsvHash.read( path )
+
+    pp rows
+
+    recs = []
+    rows.each do |row|
+      pp row
+      rec = Comparison.new
+      rec.num      = row['Num'].strip.to_i    ## remove leading n trailing whitespaces
+      rec.category = row['Category'].strip
+      rec.name     = row['Name'].strip
+
+      pp rec
+      recs << rec
+    end
+
+    self.new( recs )
+  end
+
+  def initialize( comps )
+    @comps = comps
+  end
+
+  def size() @comps.size; end
+
+  def each
+    @comps.each {|comp| yield( comp ) }
+  end
+
+  def to_a
+    @comps.collect {|comp| comp.num }   ## return array of nums   -- return something else - why? why not?
+  end
+
+end  # class Comparison
+
+end # module Factbook
+

data/lib/factbook-readers/counter.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+
+module Factbook
+
+class Counter
+
+attr_reader :data
+
+def initialize
+  @data = {}
+end
+
+def count( page )
+
+  ## walk page data hash
+  #   add nodes to data
+
+  walk( page, page.data, @data )
+end
+
+
+private
+def walk( page, hin, hout )
+   hin.each do |k,v|
+     if v.is_a? Hash
+        hout2 =  hout[k] || { count: 0, codes: '' }
+        
+        hout2[ :count ] += 1
+        
+        ## delete codes if larger (treshhold) than x (e.g. 9)
+        hout2.delete( :codes )    if hout2[ :count ] > 9
+
+        codes = hout2[ :codes ]
+        if codes    ## note: might got deleted if passed treshhold (e.g. 9 entries)
+          codes << ' '  unless codes.empty?   ## add separator (space for now)
+          codes << page.info.country_code 
+          hout2[ :codes ] = codes
+        end
+             
+        hout[k] = hout2        
+        walk( page, v, hout2 )
+     end
+   end
+end
+
+end # class Counter
+
+end # module Factbook

data/lib/factbook-readers/normalize.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+
+module Factbook  
+  module NormalizeHelper
+
+
+def normalize_category( text )
+
+  ## note: fix typos/errors with double colons e.g. note::  (instead of note:)
+
+  text = text.strip
+  text = text.sub( /:+\z/, '' )      # remove trailing : if present -- note: allow (fix) note:: too, thus, use :+
+  text = text.strip
+
+  #######################################
+  ### special cases
+
+  ##   typos e.g ntoe => use note
+  text = 'note'                         if text == 'ntoe'
+  text = 'investment in fixed capital'  if text == 'investment if fixed capital'  
+
+  ##  downcase
+  text = 'lowest point'    if text == 'Lowest point'
+  text = 'chief of state'  if text == 'Chief of state'
+
+  ##  spelling variant (use more popular one)
+  text = 'signed, but not ratified'     if text == 'signed but not ratified'
+  text = 'vectorborne diseases'         if text == 'vectorborne disease'
+  text = 'water contact disease'        if text == 'water contact diseases'
+  text = 'food or waterborne diseases'  if text == 'food or waterborne disease'
+  text = 'geographic coordinates'       if text == 'geographical coordinates'
+  text = 'note'                         if text == 'notes'
+  text = 'refugees (country of origin)' if text == 'refugees (countries of origin)'
+
+  ##    border countries (8):   -- remove (x) counter
+  text = 'border countries'   if text.start_with?( 'border countries')
+    
+  text
+end
+
+
+  end # module NormalizeHelper
+end  # module Factbook

data/lib/factbook-readers/page.rb

@@ -0,0 +1,148 @@
+
+module Factbook
+
+
+## note:
+##   some factbook pages with chrome (headers, footers, etc.)
+##     are NOT valid utf-8, thus,
+##     treat page as is (e.g. ASCII8BIT)
+#
+#   only convert to utf8 when header and footer got stripped
+
+##
+## be/benin:
+##   Key Force or FC [Lazare S?xx?HOU?xx?TO]     -- two invalid byte code chars in Political parties and leaders:
+#
+##   in Western/Windows-1252  leads to  FC [Lazare SÈHOUÉTO];
+#       Lazare Sèhouéto
+#
+#   looks good - use (assume) Windows-1252 ????
+
+##
+#   check for is ascii 7-bit ???  if yes -noworries
+#     if not, log number of chars not using ascii 7-bit
+
+
+
+class Page
+  include LogUtils::Logging
+
+  attr_reader :sects    ## "structured" access e.g. sects/subsects/etc.
+  attr_reader :info     ##  meta info e.g. country_code, country_name, region_name, last_updated, etc.
+  attr_reader :data     ## "plain" access with vanilla hash
+
+
+  ## standard version  (note: requires https)
+  SITE_BASE = 'https://www.cia.gov/library/publications/the-world-factbook/geos/{code}.html'
+
+  def initialize( code, opts={} )
+    ### keep code - why? why not??  (use page_info/info e.g. info.country_code??)
+
+    if opts[:json]
+      json = opts[:json]    ## note: json is (still) a string/text (NOT yet parsed to structured data)
+      b = JsonBuilder.from_string( json )
+    else  ## assume html
+      if opts[:html]    ## note: expects ASCII-7BIT/BINARY encoding
+         ## for debugging and testing allow "custom" passed-in html page
+        html = opts[:html]
+      else
+        url_string =  SITE_BASE.gsub( '{code}', code )
+        ## note: expects ASCII-7BIT/BINARY encoding
+
+        ## html = fetch_page( url_string )   ## use PageFetcher class - why?? why not??
+        html = Webcache.read( url_string )
+      end
+      b = Builder.from_string( html )
+    end
+
+    @sects = b.sects
+    @info  = b.info
+
+    ## todo/fix/quick hack:
+    ##  check for info opts hash entry - lets you overwrite page info
+    ##  -- use proper header to setup page info - why, why not??
+    if opts[:info]
+      info  = opts[:info]
+      @info = info
+    end
+
+    @data = {}
+    @sects.each do |sect|
+      @data[ sect.title ] = sect.data
+    end
+
+    self  ## return self (check - not needed??)
+  end
+
+
+  def to_json( opts={} )  ## convenience helper for data.to_json; note: pretty print by default!
+    if opts[:minify]
+      data.to_json
+    else
+      ## was: -- opts[:pretty] || opts[:pp]
+      JSON.pretty_generate( data )   ## note: pretty print by default!
+    end
+  end
+
+
+  def [](key)  ### convenience shortcut
+    # lets you use
+    #   page['geo']
+    #   instead of
+    #   page.data['geo']
+
+    ##  fix: use delegate data, [] from forwardable lib - why?? why not??
+
+    data[key]
+  end
+
+  ## add convenience (shortcut) accessors / attributes / fields / getters
+
+  ATTRIBUTES.each do |attrib|
+    ## e.g.
+    ##    def background()  data['Introduction']['Background']['text']; end
+    ##    def location()    data['Geography']['Location']['text'];      end
+    ##    etc.
+    if attrib.path.size == 1
+      define_method attrib.name.to_sym do
+        @data.fetch( attrib.category, {} ).
+              fetch( attrib.path[0], {} )['text']
+      end
+    else  ## assume size 2 for now
+      define_method attrib.name.to_sym do
+        @data.fetch( attrib.category, {} ).
+              fetch( attrib.path[0], {} ).
+              fetch( attrib.path[1], {} )['text']
+      end
+    end
+  end
+
+
+private
+  def fetch_page( url )
+    response = Webget.page( url )
+
+    ## note: exit on get / fetch error - do NOT continue for now - why? why not?
+    exit 1   if response.status.nok?    ## e.g.  HTTP status code != 200
+
+
+    response.text
+  end
+
+
+=begin
+def self.from_url( cc, cn )
+  html_ascii = PageFetcher.new.fetch( cc )
+  self.new( cc, cn, html_ascii )
+end
+
+def self.from_file( cc, cn, opts={} )
+  input_dir = opts[:input_dir] || '.'
+  html_ascii = File.read( "#{input_dir}/#{cc}.html" )    ## fix/todo: use ASCII8BIT/binary reader
+  self.new( cc, cn, html_ascii )
+end
+=end
+
+
+end # class Page
+end # module Factbook