factbook 2.0.0 → 2.0.1

data/data/comparisons.csv

@@ -1,75 +0,0 @@
-Num,Category,Name
-2147,Geography,Area
-2119,People and Society,Population
-2002,People and Society,Population growth rate
-2054,People and Society,Birth rate
-2066,People and Society,Death rate
-2112,People and Society,Net migration rate
-2223,People and Society,Maternal mortality rate
-2091,People and Society,Infant mortality rate
-2102,People and Society,Life expectancy at birth
-2127,People and Society,Total fertility rate
-2225,People and Society,Health expenditures
-2155,People and Society,HIV/AIDS - adult prevalence rate
-2156,People and Society,HIV/AIDS - people living with HIV/AIDS
-2157,People and Society,HIV/AIDS - deaths
-2228,People and Society,Obesity - adult prevalence rate
-2224,People and Society,Children under the age of 5 years underweight
-2206,People and Society,Education expenditures
-2229,People and Society,"Unemployment, youth ages 15-24"
-2001,Economy,GDP (purchasing power parity)
-2003,Economy,GDP - real growth rate
-2004,Economy,GDP - per capita (PPP)
-2260,Economy,Gross national saving
-2089,Economy,Industrial production growth rate
-2095,Economy,Labor force
-2129,Economy,Unemployment rate
-2172,Economy,Distribution of family income - Gini index
-2221,Economy,Taxes and other revenues
-2222,Economy,Budget surplus (+) or deficit (-)
-2186,Economy,Public debt
-2092,Economy,Inflation rate (consumer prices)
-2207,Economy,Central bank discount rate
-2208,Economy,Commercial bank prime lending rate
-2214,Economy,Stock of narrow money
-2215,Economy,Stock of broad money
-2211,Economy,Stock of domestic credit
-2200,Economy,Market value of publicly traded shares
-2187,Economy,Current account balance
-2078,Economy,Exports
-2087,Economy,Imports
-2188,Economy,Reserves of foreign exchange and gold
-2079,Economy,Debt - external
-2198,Economy,Stock of direct foreign investment - at home
-2199,Economy,Stock of direct foreign investment - abroad
-2232,Energy,Electricity - production
-2233,Energy,Electricity - consumption
-2234,Energy,Electricity - exports
-2235,Energy,Electricity - imports
-2236,Energy,Electricity - installed generating capacity
-2237,Energy,Electricity - from fossil fuels
-2239,Energy,Electricity - from nuclear fuels
-2238,Energy,Electricity - from hydroelectric plants
-2240,Energy,Electricity - from other renewable sources
-2241,Energy,Crude oil - production
-2242,Energy,Crude oil - exports
-2243,Energy,Crude oil - imports
-2244,Energy,Crude oil - proved reserves
-2245,Energy,Refined petroleum products - production
-2246,Energy,Refined petroleum products - consumption
-2247,Energy,Refined petroleum products - exports
-2248,Energy,Refined petroleum products - imports
-2249,Energy,Natural gas - production
-2250,Energy,Natural gas - consumption
-2251,Energy,Natural gas - exports
-2252,Energy,Natural gas - imports
-2253,Energy,Natural gas - proved reserves
-2150,Communications,Telephones - fixed lines
-2151,Communications,Telephones - mobile cellular
-2153,Communications,Internet users
-2053,Transportation,Airports
-2121,Transportation,Railways
-2085,Transportation,Roadways
-2093,Transportation,Waterways
-2108,Transportation,Merchant marine
-2034,Military,Military expenditures

data/lib/factbook/almanac.rb

@@ -1,72 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-
-class Almanac
-
-  ## convenience helper ("factory") 
-  def self.from_json( codes, json_dir: '.' )
-    pages = JsonPageReader.new( json_dir ).read_pages( codes )
-    self.new( pages )
-  end
-
-
-  def initialize( pages )
-    @pages = pages
-  end
-
-  def render( template )
-    buf = ''
-    @pages.each do |page|
-      text = PageCtx.new( page, template ).render
-
-      puts text     ## for debugging write country profile to console (too)
-      buf << text
-    end
-    puts "count: #{@pages.count}"
-    buf   ## return buffered almanac text
-  end
-
-
-class PageCtx
-  attr_accessor :page
-
-  def initialize(page, template)
-    @page     = page
-    @template = template
-  end
-
-  ##############################
-  ## add some "view helpers"
-
-  def name
-    ##  -- calculate name (use long name if (short) name is not availabe e.g. none)
-    ##   e.g. Austria
-    if @name.nil?
-      @name = page.name
-      @name = page.name_long  if @name == 'none'
-    end
-    @name
-  end
-
-  def names( separator: ' • ' )
-    ##  e.g. Austria • Österreich
-    if @names.nil?
-      if page.name_local.blank? || page.name_local == 'none' || page.name_local == name
-        @names = [name]    ## no local (in its own non-english language) name
-      else
-        @names = [name, page.name_local]
-      end
-    end
-    @names.join( separator )
-  end
-
-  def render
-    ERB.new( @template).result( binding )
-  end
-end   ## PageCtx
-
-end ## Almanac
-
-end # module Factbook

data/lib/factbook/attributes.rb

@@ -1,74 +0,0 @@
-# encoding: utf-8
-
-
-module Factbook
-
-class Attributes
-
-  Attribute = Struct.new( :name,     
-                          :category,  ## e.g. Introduction, Geography, etc.
-                          :path,      ## note: is an array  e.g. ["Area - comparative"] or ["Area", "land"] etc.
-                        )
-
-  def self.from_yaml( path )
-
-    h = YAML.load_file( path )
-    pp h
-
-    attribs = []
-
-    ## note: use a copy (e.g. h.dup) for now (hash gets changed by build_attribs!!)
-    new_h = h.dup
-    new_h.each do |k,v|
-      category = k
-      build_attribs( attribs, category, [], v )
-    end
-
-    self.new( attribs )
-  end
-
-
-  def self.build_attribs( attribs, category, path, h )  
-     
-      ## assume it's an attribute definition hash
-      ##   note: !! exclude special cases:
-      ##      Capital           -- incl. name key itself
-      ##      National anthem
-     if h.has_key?( 'name' ) &&  ['Capital','National anthem'].include?( path[-1] ) == false
-       a = Attribute.new
-       a.name     = h['name']
-       a.category = category
-       a.path     = path
-
-       puts "  adding attribute >#{a.name}< using #{a.category} / #{a.path.inspect}"       
-       attribs << a
-       
-       ## note: make sure a modifable copy (of h) gets passed in
-       h.delete( 'name' )
-     end
-
-     return  if h.empty?    ## empty hash; nothing (more) to do; return     
-     
-     ## continue walking (recursive)
-     h.each do |k,v|
-       new_path = path.dup << k   ## note: create a new array (copy) 
-       build_attribs( attribs, category, new_path, v )
-    end
-  end   
-
-
-  def initialize( attribs )
-    @attribs = attribs
-  end
-  
-  def to_a() @attribs; end
-  def size() @attribs.size; end
-
-  def each
-    @attribs.each { |attrib| yield( attrib ) }
-  end
-
-end  # class Attributes
-
-end # module Factbook
-

data/lib/factbook/builder.rb

@@ -1,212 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-class Builder     ## todo: change to PageBuilder ???
-  include LogUtils::Logging
-
-
-=begin
-def self.from_cc( cc, opts={} )  ## rename to from_file_for_country() or from_file_for_cc() or something - why?? why not??
-  ## check/todo: rename input_dir to just dir or to include ?
-  ##   (there's no output_dir)?? - why? why not?
-  input_dir = opts[:input_dir] || '.'
-  self.from_file( "#{input_dir}/#{cc}.html" )
-end
-=end
-
-
-def self.from_file( path )
-  html_ascii = File.read( path )    ## fix/todo: use ASCII8BIT/binary reader !!!!!
-  self.from_string( html_ascii )
-end
-
-def self.from_string( html_ascii )   ## note: expects ASCII-7BIT/BINARY encoding
-  self.new( html_ascii )
-end
-
-
-attr_reader :html_ascii,     ## full "original" 1:1 page in "original/ascii8/binary" encoding
-            :html,           ## utf-8 encoded profile
-            :html_debug,     ## html w/ mapping markers - rename to html_markers - why? why not?
-            :info,            ## page info incl. country_name, region_name, last_updated etc.
-            :errors,          ## encoding erros etc.
-            :sects
-
-
-def initialize( html_ascii )
-  @html_ascii = html_ascii
-
-  ## todo/fix: use/assume windows 12xx?? encoding - change encoding to utf-8  (from binary/ascii8bit)
-  @html, @info, @errors = Sanitizer.new.sanitize( @html_ascii )
-
-
-  html_sects =  if @html.empty?
-                   ## note: support "empty" pages - old format waiting for update!!!
-                   ##    cannot parse for now
-                   []  ## return empty (no) sections for now - sorry (its just one page with code cc anyway!!)
-                else
-                   @html_debug = map_sects( @html )
-                   @html_debug = map_subsects( @html_debug )
-
-                   split_sects( @html_debug )
-                end
-
-  pp html_sects
-
-  ## debug
-  ## File.open( 'tmp/br.debug.html', 'w:utf-8') { |f| f.write( @html_debug ) }
-
-
-  @sects = []
-  html_sects.each do |html_sect|
-    html_sect_head = html_sect[0]
-    html_subsects  = html_sect[1]
-    puts html_sect_head
-    puts html_subsects.size
-
-    ## get section title
-    ##  @SECTION{Economy}  => Economy
-    if html_sect_head =~ /@SECTION{(.+?)}/
-      title = $1.strip
-      puts title
-      sect = Sect.new
-      sect.title = title
-      ## get subsections
-      subsects = []
-      html_subsects.each do |html_subsect|
-        html_subsect_head = html_subsect[0]
-        html_subsect_body = html_subsect[1]
-        if html_subsect_head =~ /@SUBSECTION{(.+?)}/
-          title = $1.strip
-          title = title.sub( /:\z/, '' )    # remove trailing : if present
-          title = title.strip
-
-          puts title
-          subsect = Subsect.new
-          subsect.title = title     ## todo/fix: cut off trailing colon (:)
-
-          b = Factbook::ItemBuilder.new( html_subsect_body, title )
-          h = b.read
-          subsect.data = h
-
-          subsects << subsect
-        else
-          ## warn/fix: no subsection title found
-        end
-      end
-      sect.subsects = subsects
-      @sects << sect
-    else
-      ## warn/fix:  no section title found
-    end
-  end
-
-  self  ## return self -- needed?? default (standard) anyway?? check and remove
-end
-
-
-
-def map_sects( html )
-   ## convert section titles to "unified" marker
-   ## e.g.
-   ##   <h2>Introduction</h2>
-
-  title_regex= /<h2>
-                 \s*
-                   (.+?)  ## note: use non-greedy; do NOT allow tags inside for now
-                 \s*
-                <\/h2>
-              /xim
-
-  html = html.gsub( title_regex ) do |m|
-     puts "** found section >#{$1}<:"
-     puts "   >|#{m}|<"
-
-     "\n\n@SECTION{#{$1}}\n\n"
-  end
-  html
-end
-
-
-def map_subsects( html )
-   ## convert subsection titles to "unified" marker
-   ## e.g.
-   ##  <h3>Disputes - international:</h3>
-
-  title_regex= /<h3>
-                  \s*
-                   (.+?)                ## note: use non-greedy; allows tags inside - why? why not
-                  \s*
-                 <\/h3>
-               /xim
-
-  html = html.gsub( title_regex ) do |m|
-     puts "** found subsection >#{$1}<:"
-     puts "   >|#{m}|<"
-
-     "\n@SUBSECTION{#{$1}}\n"
-  end
-  html
-end
-
-
-
-def split_sects( html )
-  ####
-  #  split html in sections (divided by section headings)
-  #  e.g. remove optional prolog ??,
-  ##   [[heading,sect],
-  ##    [heading,sect],
-  ##    [heading,sect],...]
-
-  ## note: "wrap" regex in a capture group (just one)
-  ##   String#split will include all catpure groups in the result array
-
-  section_regex= /(@SECTION{.+?})/  ## note: use non-greedy -- check: need to escape {} ??
-
-  chunks = html.split( section_regex )
-
-  ## check if first item is a section or (html) prolog
-  #   if prolog (remove)
-  chunks.slice!(0)  unless chunks[0] =~ /@SECTION/  ## starts w/ @SECTION
-
-  pairs = chunks.each_slice(2).to_a
-
-  ## now split subsections
-  newpairs = []
-  pairs.each do |item|
-    ## todo: after cleanup prolog; remove @SECTION{} ?? - just keep title - why, why not??
-    newpairs << [item[0], split_subsects( item[1]) ]
-  end
-  newpairs
-end
-
-
-def split_subsects( html )
-  ####
-  #  split html in subsections (divided by subsection headings)
-  #  e.g. remove optional prolog ??,
-  ##   [[heading,sect],
-  ##    [heading,sect],
-  ##    [heading,sect],...]
-
-  ## note: "wrap" regex in a capture group (just one)
-  ##   String#split will include all catpure groups in the result array
-
-  subsection_regex= /(@SUBSECTION{.+?})/  ## note: use non-greedy -- check: need to escape {} ??
-
-  chunks = html.split( subsection_regex )
-
-  ## check if first item is a section or (html) prolog
-  #   if prolog (remove)
-  chunks.slice!(0)  unless chunks[0] =~ /@SUBSECTION/  ## starts w/ @SUBSECTION
-
-  pairs = chunks.each_slice(2).to_a
-  pairs
-end
-
-end # class Builder
-
-
-end # module Factbook

data/lib/factbook/builder_item.rb

@@ -1,126 +0,0 @@
-# encoding: utf-8
-
-module Factbook
-
-class ItemBuilder       ## renameto ItemReader, ItemParser - why? why not??
-  include LogUtils::Logging
-  include NormalizeHelper    ##  e.g. normalize_category
-
-def initialize( html, name )
-  @html = html
-  @name = name     # add category/field name e.g. Area, Location, etc.
-end
-
-
-
-##
-## <div class="category_data subfield text">
-## Portuguese  (official and most widely spoken language)
-##
-## </div>
-## <div class="category_data note">
-## <p><strong>note:</strong> less common languages include Spanish (border areas and schools), German, Italian, Japanese, English, and a large number of minor Amerindian languages</p>
-## </div>
-
-
-def read
-  ## return hash from html snippet
-  doc = Nokogiri::HTML.fragment( @html )
-
-  data = {}
-
-  ## note:
-  ##   skip whitespace text nodes (e.g. \n\n etc); just use divs
-  doc_children = doc.children.filter('div')
-
-  puts "  parsing >#{@name}< - #{doc_children.size} category_data divs(s):"
-
-  doc_children.each_with_index do |div,i|
-    if div['class'].index( 'note' )
-      text = squish( div.text.strip )
-      puts "category_data: >#{text}<"
-
-      data['note'] = { 'text' => text }
-    elsif div['class'].index( 'historic' )
-      ## add all historic together into one for now
-        text = squish( div.text.strip )
-        puts "category_data: >#{text}<"
-
-        if i == 0
-          data['text'] = text
-        else
-          ## append with / for now
-          data['text'] += " / #{text}"
-        end
-      elsif div.css( 'span.subfield-name').empty?
-      ## assume "implied text field"
-      ## check for index == 1 / child count == 1 - why? why not
-      text = squish( div.text.strip )    ## fix/todo: use strip
-      puts "category_data: >#{text}<"
-
-      data['text'] = text
-
-      ## must be always first node for now
-      if i != 0
-        puts "!! ERROR - 'implied' category W/O name NOT first div / node:"
-        puts @html
-        exit 1
-      end
-    elsif div['class'].index( 'grouped_subfield' )
-## split grouped subfield!!
-##   <span class="subfield-name">arable land:</span>
-## <span class="subfield-number">8.6%</span>
-## <span class="subfield-date">(2011 est.)</span>
-##  /
-## <span class="subfield-name">permanent crops:</span>
-## <span class="subfield-number">0.8%</span>
-## <span class="subfield-date">(2011 est.)</span>
-##   /
-## <span class="subfield-name">permanent pasture:</span>
-## <span class="subfield-number">23.5%</span>
-## <span class="subfield-date">(2011 est.)</span>
-
-## join names for now - why? why not?
-##  e.g. becomes:
-##   arable land / permanent crops / permanent pasture: for key ??
-     span_names = div.css( 'span.subfield-name')
-     keys = []
-     span_names.each do |span|
-       keys << normalize_category( span.text.strip )
-       span.replace( '' )
-     end
-     key = keys.join( ' / ')
-     text = squish( div.text.strip )
-     puts "category_data key >#{key}<: >#{text}<"
-     data[ key ] = { 'text' => text }
-    else
-      ## get subfield name
-      span_names = div.css( 'span.subfield-name')
-      if span_names.size > 1
-        puts "!! ERROR - found more than one subfield-name:"
-        puts div.to_html
-        exit 1
-      end
-      key = normalize_category( span_names[0].text.strip )
-      span_names[0].replace( '' )
-
-      text = squish( div.text.strip )
-      puts "category_data key >#{key}<: >#{text}<"
-      data[ key ] = { 'text' => text }
-    end
-  end
-
-
-  pp data
-  data
-end
-
-
-
-def squish( str )
-  str.gsub( /[ \t\n\r]{2,}/, ' ')  ## replace multi-spaces (incl. newlines with once space)
-end
-
-end # class ItemBuilder
-
-end # module Factbook