factbook-readers 1.0.1 → 1.1.0

data/data/comparisons.csv

@@ -1,75 +0,0 @@
-Num,Category,Name
-2147,Geography,Area
-2119,People and Society,Population
-2002,People and Society,Population growth rate
-2054,People and Society,Birth rate
-2066,People and Society,Death rate
-2112,People and Society,Net migration rate
-2223,People and Society,Maternal mortality rate
-2091,People and Society,Infant mortality rate
-2102,People and Society,Life expectancy at birth
-2127,People and Society,Total fertility rate
-2225,People and Society,Health expenditures
-2155,People and Society,HIV/AIDS - adult prevalence rate
-2156,People and Society,HIV/AIDS - people living with HIV/AIDS
-2157,People and Society,HIV/AIDS - deaths
-2228,People and Society,Obesity - adult prevalence rate
-2224,People and Society,Children under the age of 5 years underweight
-2206,People and Society,Education expenditures
-2229,People and Society,"Unemployment, youth ages 15-24"
-2001,Economy,GDP (purchasing power parity)
-2003,Economy,GDP - real growth rate
-2004,Economy,GDP - per capita (PPP)
-2260,Economy,Gross national saving
-2089,Economy,Industrial production growth rate
-2095,Economy,Labor force
-2129,Economy,Unemployment rate
-2172,Economy,Distribution of family income - Gini index
-2221,Economy,Taxes and other revenues
-2222,Economy,Budget surplus (+) or deficit (-)
-2186,Economy,Public debt
-2092,Economy,Inflation rate (consumer prices)
-2207,Economy,Central bank discount rate
-2208,Economy,Commercial bank prime lending rate
-2214,Economy,Stock of narrow money
-2215,Economy,Stock of broad money
-2211,Economy,Stock of domestic credit
-2200,Economy,Market value of publicly traded shares
-2187,Economy,Current account balance
-2078,Economy,Exports
-2087,Economy,Imports
-2188,Economy,Reserves of foreign exchange and gold
-2079,Economy,Debt - external
-2198,Economy,Stock of direct foreign investment - at home
-2199,Economy,Stock of direct foreign investment - abroad
-2232,Energy,Electricity - production
-2233,Energy,Electricity - consumption
-2234,Energy,Electricity - exports
-2235,Energy,Electricity - imports
-2236,Energy,Electricity - installed generating capacity
-2237,Energy,Electricity - from fossil fuels
-2239,Energy,Electricity - from nuclear fuels
-2238,Energy,Electricity - from hydroelectric plants
-2240,Energy,Electricity - from other renewable sources
-2241,Energy,Crude oil - production
-2242,Energy,Crude oil - exports
-2243,Energy,Crude oil - imports
-2244,Energy,Crude oil - proved reserves
-2245,Energy,Refined petroleum products - production
-2246,Energy,Refined petroleum products - consumption
-2247,Energy,Refined petroleum products - exports
-2248,Energy,Refined petroleum products - imports
-2249,Energy,Natural gas - production
-2250,Energy,Natural gas - consumption
-2251,Energy,Natural gas - exports
-2252,Energy,Natural gas - imports
-2253,Energy,Natural gas - proved reserves
-2150,Communications,Telephones - fixed lines
-2151,Communications,Telephones - mobile cellular
-2153,Communications,Internet users
-2053,Transportation,Airports
-2121,Transportation,Railways
-2085,Transportation,Roadways
-2093,Transportation,Waterways
-2108,Transportation,Merchant marine
-2034,Military,Military expenditures

data/lib/factbook-readers/builder.rb

@@ -1,187 +0,0 @@
-
-module Factbook
-
-class Builder     ## todo: change to HtmlBuilder or PageBuilder ???
-  include LogUtils::Logging
-
-
-
-attr_reader :html_original,    ## full "original" 1:1 page
-            :html,             ## cut-out and sanitized profile
-            :html_debug,      ## html w/ mapping markers - rename to html_markers - why? why not?
-            :info,            ## page info incl. country_name, region_name, last_updated etc.
-            :errors,          ## encoding erros etc.
-            :sects
-
-
-def initialize( html_original )
-  @html_original = html_original
-
-  @html, @info, @errors = Sanitizer.new.sanitize( @html_original )
-
-
-  html_sects =  if @html.empty?
-                   ## note: support "empty" pages - old format waiting for update!!!
-                   ##    cannot parse for now
-                   @html_debug = ''
-                   []  ## return empty (no) sections for now - sorry (its just one page with code cc anyway!!)
-                else
-                   @html_debug = map_sects( @html )
-                   @html_debug = map_subsects( @html_debug )
-
-                   split_sects( @html_debug )
-                end
-
-  pp html_sects
-
-  ## debug
-  ##   File.open( 'tmp/br.debug.html', 'w:utf-8') { |f| f.write( @html_debug ) }
-
-
-  @sects = []
-  html_sects.each do |html_sect|
-    html_sect_head = html_sect[0]
-    html_subsects  = html_sect[1]
-    puts html_sect_head
-    puts html_subsects.size
-
-    ## get section title
-    ##  @SECTION{Economy}  => Economy
-    if html_sect_head =~ /@SECTION{(.+?)}/
-      title = $1.strip
-      puts title
-      sect = Sect.new
-      sect.title = title
-      ## get subsections
-      subsects = []
-      html_subsects.each do |html_subsect|
-        html_subsect_head = html_subsect[0]
-        html_subsect_body = html_subsect[1]
-        if html_subsect_head =~ /@SUBSECTION{(.+?)}/
-          title = $1.strip
-          title = title.sub( /:\z/, '' )    # remove trailing : if present
-          title = title.strip
-
-          puts title
-          subsect = Subsect.new
-          subsect.title = title     ## todo/fix: cut off trailing colon (:)
-
-          b = Factbook::ItemBuilder.new( html_subsect_body, title )
-          h = b.read
-          subsect.data = h
-
-          subsects << subsect
-        else
-          ## warn/fix: no subsection title found
-        end
-      end
-      sect.subsects = subsects
-      @sects << sect
-    else
-      ## warn/fix:  no section title found
-    end
-  end
-end
-
-
-H2_RE = /<h2>
-          \s*
-         (.+?)  ## note: use non-greedy; do NOT allow tags inside for now
-          \s*
-         <\/h2>
-        /xim
-
-def map_sects( html )
-   ## convert section titles to "unified" marker
-   ## e.g.
-   ##   <h2>Introduction</h2>
-
-  html = html.gsub( H2_RE ) do |m|
-     puts "** found section >#{$1}<:"
-     puts "   >|#{m}|<"
-
-     "\n\n@SECTION{#{$1}}\n\n"
-  end
-  html
-end
-
-
-H3_RE = /<h3>
-          \s*
-         (.+?)                ## note: use non-greedy; allows tags inside - why? why not
-          \s*
-        <\/h3>
-       /xim
-
-def map_subsects( html )
-   ## convert subsection titles to "unified" marker
-   ## e.g.
-   ##  <h3>Disputes - international:</h3>
-
-  html = html.gsub( H3_RE ) do |m|
-     puts "** found subsection >#{$1}<:"
-     puts "   >|#{m}|<"
-
-     "\n@SUBSECTION{#{$1}}\n"
-  end
-  html
-end
-
-
-
-def split_sects( html )
-  ####
-  #  split html in sections (divided by section headings)
-  #  e.g. remove optional prolog ??,
-  ##   [[heading,sect],
-  ##    [heading,sect],
-  ##    [heading,sect],...]
-
-  ## note: "wrap" regex in a capture group (just one)
-  ##   String#split will include all catpure groups in the result array
-
-  ## note: use non-greedy -- check: need to escape {} ??
-  chunks = html.split( /(@SECTION{.+?})/ )
-
-  ## check if first item is a section or (html) prolog
-  #   if prolog (remove)
-  chunks.slice!(0)  unless chunks[0] =~ /@SECTION/  ## starts w/ @SECTION
-
-  pairs = chunks.each_slice(2).to_a
-
-  ## now split subsections
-  newpairs = []
-  pairs.each do |item|
-    ## todo: after cleanup prolog; remove @SECTION{} ?? - just keep title - why, why not??
-    newpairs << [item[0], split_subsects( item[1]) ]
-  end
-  newpairs
-end
-
-
-def split_subsects( html )
-  ####
-  #  split html in subsections (divided by subsection headings)
-  #  e.g. remove optional prolog ??,
-  ##   [[heading,sect],
-  ##    [heading,sect],
-  ##    [heading,sect],...]
-
-  ## note: "wrap" regex in a capture group (just one)
-  ##   String#split will include all catpure groups in the result array
-
-  ## note: use non-greedy -- check: need to escape {} ??
-  chunks = html.split( /(@SUBSECTION{.+?})/ )
-
-  ## check if first item is a section or (html) prolog
-  #   if prolog (remove)
-  chunks.slice!(0)  unless chunks[0] =~ /@SUBSECTION/  ## starts w/ @SUBSECTION
-
-  pairs = chunks.each_slice(2).to_a
-  pairs
-end
-
-end # class Builder
-
-
-end # module Factbook

data/lib/factbook-readers/builder_item.rb

@@ -1,201 +0,0 @@
-
-module Factbook
-
-class ItemBuilder       ## renameto ItemReader, ItemParser - why? why not??
-  include LogUtils::Logging
-  include NormalizeHelper    ##  e.g. normalize_category
-
-def initialize( html, name )
-  @html = html
-  @name = name     # add category/field name e.g. Area, Location, etc.
-end
-
-
-
-##
-## <div class="category_data subfield text">
-## Portuguese  (official and most widely spoken language)
-##
-## </div>
-## <div class="category_data note">
-## <p><strong>note:</strong> less common languages include Spanish (border areas and schools), German, Italian, Japanese, English, and a large number of minor Amerindian languages</p>
-## </div>
-
-
-def read
-  ## return hash from html snippet
-  doc = Nokogiri::HTML.fragment( @html )
-
-  data = {}
-
-  ## note:
-  ##   skip whitespace text nodes (e.g. \n\n etc); just use divs
-  doc_children = doc.children.filter('div')
-
-  puts "  parsing >#{@name}< - #{doc_children.size} category_data divs(s):"
-
-  ## hanlde special case for
-  ##  multiple 'grouped_subfield' first
-  ##  e.g. used in
-  ##   - Drinking water source:
-  ##   - Sanitation facility access:
-
-  grouped_children = []
-  other_children   = []
-
-  doc_children.each do |div|
-     if div['class'] && div['class'].index( 'grouped_subfield' )
-        grouped_children << div
-     else
-        other_children << div
-     end
-  end
-
-
-  ## note: only use special rule if more than one div marked grouped_
-  if grouped_children.size > 1
-    ## continue processing the rest as usual
-    doc_children =  other_children
-
-    key = nil
-    grouped_children.each do |div|
-       if !div.css( 'span.subfield-group').empty?
-         # start a new group
-         span_group = div.at( 'span.subfield-group')
-         key  = normalize_category( span_group.text.strip )
-         span_group.replace( '' )
-
-         text = squish( div.text.strip )
-         puts "new group - category_data key >#{key}<: >#{text}<"
-         data[ key ] = { 'text' => text }
-       else
-         ## append to (last) group
-         text = squish( div.text.strip )
-         puts "add group - category_data key >#{key}<: >#{text}<"
-         data[ key ]['text'] += " / #{text}"
-       end
-    end
-  end
-
-
-doc_children.each_with_index do |div,i|
-  if div['class'] && div['class'].index( 'category_data' )
-    if div['class'].index( 'note' )
-      text = squish( div.text.strip )
-      puts "category_data: >#{text}<"
-
-      ## note: for now only allow one note per subsection/field data block
-      if data['note']
-        puts "!! ERROR: note already taken:"
-        puts data['note']
-        puts  div.to_html
-        exit 1
-      end
-
-      ## note: add note directly (that is, W/O extra hash and text node/key)
-      data['note'] = text
-    elsif div['class'].index( 'historic' )
-      ## add all historic together into one for now
-        text = squish( div.text.strip )
-        puts "category_data: >#{text}<"
-
-        if data['text']
-          ## append with / for now
-          data['text'] += " / #{text}"
-        else
-          data['text'] = text
-          ## check if history is first node
-          if i != 0
-            puts "!! ERROR: expected first historic node to be first node but it is #{i+1}:"
-            puts div.to_html
-            exit 1
-          end
-        end
-      elsif div.css( 'span.subfield-name').empty?
-        ## assume "implied text field"
-        ## check for index == 1 / child count == 1 - why? why not
-        text = squish( div.text.strip )    ## fix/todo: use strip
-        puts "category_data: >#{text}<"
-
-        data['text'] = text
-
-        ## must be always first node for now
-        if i != 0
-          puts "!! ERROR - 'implied' category W/O name NOT first div / node:"
-          puts div.to_html
-          exit 1
-        end
-    elsif div['class'].index( 'grouped_subfield' )
-## split grouped subfield!!
-##   <span class="subfield-name">arable land:</span>
-## <span class="subfield-number">8.6%</span>
-## <span class="subfield-date">(2011 est.)</span>
-##  /
-## <span class="subfield-name">permanent crops:</span>
-## <span class="subfield-number">0.8%</span>
-## <span class="subfield-date">(2011 est.)</span>
-##   /
-## <span class="subfield-name">permanent pasture:</span>
-## <span class="subfield-number">23.5%</span>
-## <span class="subfield-date">(2011 est.)</span>
-
-## join names for now - why? why not?
-##  e.g. becomes:
-##   arable land / permanent crops / permanent pasture: for key ??
-     span_names = div.css( 'span.subfield-name')
-     keys = []
-     span_names.each do |span|
-       keys << normalize_category( span.text.strip )
-       span.replace( '' )
-     end
-     key = keys.join( ' / ')
-     text = squish( div.text.strip )
-     puts "category_data key >#{key}<: >#{text}<"
-     data[ key ] = { 'text' => text }
-    else
-      ## get subfield name
-      span_names = div.css( 'span.subfield-name')
-      if span_names.size > 1
-        puts "!! ERROR - found more than one subfield-name:"
-        puts div.to_html
-        exit 1
-      end
-      key = normalize_category( span_names[0].text.strip )
-      span_names[0].replace( '' )
-
-      text = squish( div.text.strip )
-      puts "category_data key >#{key}<: >#{text}<"
-      data[ key ] = { 'text' => text }
-    end
-  else
-      text = squish( div.text.strip )
-      if text =~ /country\s+
-                  comparison\s+
-                  to\s+
-                  the\s+
-                  world:\s+
-                  ([0-9]+)/xim
-        data[ 'country comparison to the world' ] = $1.to_i
-      else
-        puts "!! ERROR: div (W/O category_data class):"
-        puts div.to_html
-        exit 1
-      end
-  end
-end
-
-
-  pp data
-  data
-end
-
-
-
-
-def squish( str )
-  str.gsub( /[ \t\n\r]{2,}/, ' ')  ## replace multi-spaces (incl. newlines with once space)
-end
-
-end # class ItemBuilder
-
-end # module Factbook