factbook 0.1.2 → 0.1.3

data/Manifest.txt

@@ -15,6 +15,7 @@ test/data/countrytemplate_mx.html
 test/data/countrytemplate_vt.html
 test/data/countrytemplate_xx.html
 test/helper.rb
+test/test_fields.rb
 test/test_json.rb
 test/test_page.rb
 test/test_page_old.rb

data/README.md

@@ -21,44 +21,460 @@ offers free country profiles in the public domain (that is, no copyright(s), no
 
 ### Get country profile page as a hash (that is, structured data e.g. nested key/values)
 
-    page = Factbook::Page.new( 'br' )   # br is the country code for Brazil
-    pp page.data                        # pretty print hash
+```ruby
 
+page = Factbook::Page.new( 'br' )   # br is the country code for Brazil
+pp page.data                        # pretty print hash
+
+```
 
 ### Save to disk as JSON
 
-    page = Factbook::Page.new( 'br' )
-    File.open( 'br.json', 'w') do |f|
-      f.write page.to_json( pretty: true )
-    end
+```ruby
+
+page = Factbook::Page.new( 'br' )
+File.open( 'br.json', 'w') do |f|
+  f.write page.to_json( pretty: true )
+end
+
+```
+
+### Options - Header, "Long" Category / Field Names
+
+#### Include Header Option - `header: true`
+
+```ruby
+page = Factbook::Page.new( 'br', header: true )
+```
+
+will include a leading header section. Example:
+
+```json
+{
+ "Header": {
+   "code": "au",
+   "generator": "factbook/0.1.2",
+   "last_built": "2014-08-24 12:55:39 +0200"
+ }
+ ...
+}
+```
+
+#### "Long" Category / Field Names Option - `fields: 'long'`
+
+```ruby
+page = Factbook::Page.new( 'br', fields: 'long')
+```
+
+will change the category / field names to the long form  (that is, passing through unchanged from the source).
+e.g.
+
+```ruby
+page['econ']['budget_surplus_or_deficit']['text']
+page['econ']['labor_force_by_occupation']['agriculture']
+page['trans']['ports_and_terminals']['river_ports']
+```
+becomes
+
+```ruby
+page['Economy']['Budget surplus (+) or deficit (-)']['text']
+page['Economy']['Labor force - by occupation']['agriculture']
+page['Transportation']['Ports and terminals']['river port(s)']
+```
+
+Note: You can - of course - use the options together e.g.
+
+```ruby
+page = Factbook::Page.new( 'br', header: true, fields: 'long' )
+```
+
+or
+
+```ruby
+opts = {
+  header: true,
+  fields: 'long'
+}
+page = Factbook::Page.new( 'br', opts )
+```
+
+
+## The World Factbook Summary (267 Entries)
+
+The World Factbook includes 267 entries -
+195 sovereign countries /
+2 others /
+58 dependencies /
+6 miscellaneous /
+5 oceans /
+1 world:
+
+
+### Sovereign Countries (195)
+
+**A**
+`af` Afghanistan
+`al` Albania
+`ag` Algeria
+`an` Andorra
+`ao` Angola
+`ac` Antigua and Barbuda
+`ar` Argentina
+`am` Armenia
+`as` Australia
+`au` Austria
+`aj` Azerbaijan
+**B**
+`bf` The Bahamas
+`ba` Bahrain
+`bg` Bangladesh
+`bb` Barbados
+`bo` Belarus
+`be` Belgium
+`bh` Belize
+`bn` Benin
+`bt` Bhutan
+`bl` Bolivia
+`bk` Bosnia and Herzegovina
+`bc` Botswana
+`br` Brazil
+`bx` Brunei
+`bu` Bulgaria
+`uv` Burkina Faso
+`bm` Burma
+`by` Burundi
+**C**
+`cb` Cambodia
+`cm` Cameroon
+`ca` Canada
+`cv` Cape Verde
+`ct` Central African Republic
+`cd` Chad
+`ci` Chile
+`ch` China
+`co` Colombia
+`cn` Comoros
+`cg` Congo DR
+`cf` Congo
+`cs` Costa Rica
+`iv` Cote d'Ivoire
+`hr` Croatia
+`cu` Cuba
+`cy` Cyprus
+`ez` Czech Republic
+**D**
+`da` Denmark
+`dj` Djibouti
+`do` Dominica
+`dr` Dominican Republic
+**E**
+`ec` Ecuador
+`eg` Egypt
+`es` El Salvador
+`ek` Equatorial Guinea
+`er` Eritrea
+`en` Estonia
+`et` Ethiopia
+**F**
+`fj` Fiji
+`fi` Finland
+`fr` France
+**G**
+`gb` Gabon
+`ga` The Gambia
+`gg` Georgia
+`gm` Germany
+`gh` Ghana
+`gr` Greece
+`gj` Grenada
+`gt` Guatemala
+`gv` Guinea
+`pu` Guinea-Bissau
+`gy` Guyana
+**H**
+`ha` Haiti
+`ho` Honduras
+`hu` Hungary
+**I**
+`ic` Iceland
+`in` India
+`id` Indonesia
+`ir` Iran
+`iz` Iraq
+`ei` Ireland
+`is` Israel
+`it` Italy
+**J**
+`jm` Jamaica
+`ja` Japan
+`jo` Jordan
+**K**
+`kz` Kazakhstan
+`ke` Kenya
+`kr` Kiribati
+`kn` North Korea
+`ks` South Korea
+`kv` Kosovo
+`ku` Kuwait
+`kg` Kyrgyzstan
+**L**
+`la` Laos
+`lg` Latvia
+`le` Lebanon
+`lt` Lesotho
+`li` Liberia
+`ly` Libya
+`ls` Liechtenstein
+`lh` Lithuania
+`lu` Luxembourg
+**M**
+`mk` Macedonia
+`ma` Madagascar
+`mi` Malawi
+`my` Malaysia
+`mv` Maldives
+`ml` Mali
+`mt` Malta
+`rm` Marshall Islands
+`mr` Mauritania
+`mp` Mauritius
+`mx` Mexico
+`fm` Micronesia
+`md` Moldova
+`mn` Monaco
+`mg` Mongolia
+`mj` Montenegro
+`mo` Morocco
+`mz` Mozambique
+**N**
+`wa` Namibia
+`nr` Nauru
+`np` Nepal
+`nl` Netherlands
+`nz` New Zealand
+`nu` Nicaragua
+`ng` Niger
+`ni` Nigeria
+`no` Norway
+**O**
+`mu` Oman
+**P**
+`pk` Pakistan
+`ps` Palau
+`pm` Panama
+`pp` Papua New Guinea
+`pa` Paraguay
+`pe` Peru
+`rp` Philippines
+`pl` Poland
+`po` Portugal
+**Q**
+`qa` Qatar
+**R**
+`ro` Romania
+`rs` Russia
+`rw` Rwanda
+**S**
+`sc` Saint Kitts and Nevis
+`st` Saint Lucia
+`vc` Saint Vincent and the Grenadines
+`ws` Samoa
+`sm` San Marino
+`tp` Sao Tome and Principe
+`sa` Saudi Arabia
+`sg` Senegal
+`ri` Serbia
+`se` Seychelles
+`sl` Sierra Leone
+`sn` Singapore
+`lo` Slovakia
+`si` Slovenia
+`bp` Solomon Islands
+`so` Somalia
+`sf` South Africa
+`od` South Sudan
+`sp` Spain
+`ce` Sri Lanka
+`su` Sudan
+`ns` Suriname
+`wz` Swaziland
+`sw` Sweden
+`sz` Switzerland
+`sy` Syria
+**T**
+`ti` Tajikistan
+`tz` Tanzania
+`th` Thailand
+`tt` Timor-Leste
+`to` Togo
+`tn` Tonga
+`td` Trinidad and Tobago
+`ts` Tunisia
+`tu` Turkey
+`tx` Turkmenistan
+`tv` Tuvalu
+**U**
+`ug` Uganda
+`up` Ukraine
+`ae` United Arab Emirates
+`uk` United Kingdom
+`us` United States
+`uy` Uruguay
+`uz` Uzbekistan
+**V**
+`nh` Vanuatu
+`vt` Vatican City (Holy See)
+`ve` Venezuela
+`vm` Vietnam
+**Y**
+`ym` Yemen
+**Z**
+`za` Zambia
+`zi` Zimbabwe
+
+
+### Other (2)
+
+`tw` Taiwan
+`ee` European Union
+
+### Dependencies (58)
+
+Australia (6):
+`at` Ashmore and Cartier Islands
+`kt` Christmas Island
+`ck` Cocos (Keeling) Islands
+`cr` Coral Sea Islands
+`hm` Heard Island and McDonald Islands
+`nf` Norfolk Island
+
+China (2):
+`hk` Hong Kong
+`mc` Macau
+
+Denmark (2):
+`fo` Faroe Islands
+`gl` Greenland
+
+France (8):
+`ip` Clipperton Island
+`fp` French Polynesia
+`fs` French Southern and Antarctic Lands
+`nc` New Caledonia
+`tb` Saint Barthelemy
+`rn` Saint Martin
+`sb` Saint Pierre and Miquelon
+`wf` Wallis and Futuna
+
+Netherlands (3):
+`aa` Aruba
+`uc` Curacao
+`nn` Sint Maarten
+
+New Zealand (3):
+`cw` Cook Islands
+`ne` Niue
+`tl` Tokelau
+
+Norway (3):
+`bv` Bouvet Island
+`jn` Jan Mayen
+`sv` Svalbard
+
+Great Britain (17):
+`ax` Akrotiri (Sovereign Base)
+`av` Anguilla
+`bd` Bermuda
+`io` British Indian Ocean Territory
+`vi` British Virgin Islands
+`cj` Cayman Islands
+`dx` Dhekelia (Sovereign Base)
+`fk` Falkland Islands
+`gi` Gibraltar
+`gk` Guernsey
+`je` Jersey
+`im` Isle of Man
+`mh` Montserrat
+`pc` Pitcairn Islands
+`sh` Saint Helena
+`sx` South Georgia and the South Sandwich Islands
+`tk` Turks and Caicos Islands
+
+United States (14):
+`aq` American Samoa
+`gq` Guam
+`bq` Navassa Island
+`cq` Northern Mariana Islands
+`rq` Puerto Rico
+`vq` US Virgin Islands
+`wq` Wake Island
+`um` US Pacific Island Wildlife Refuges
+(Baker Island, Howland Island, Jarvis Island, Johnston Atoll, Kingman Reef, Midway Islands, Palmyra Atoll)
+
+
+### Miscellaneous (6)
+
+`ay` Antarctica
+`gz` Gaza Strip
+`pf` Paracel Islands
+`pg` Spratly Islands
+`we` West Bank
+`wi` Western Sahara
+
+### Oceans (5)
+
+`xq` Arctic Ocean
+`zh` Atlantic Ocean
+`xo` Indian Ocean
+`zn` Pacific Ocean
+`oo` Southern Ocean
+
+### World (1)
+
+`xx` World
+
+
+
+
+## Ready-To-Use Public Domain Factbook Datasets
 
+[openmundi/factbook.json](https://github.com/openmundi/factbook.json) - open (public domain)
+factbook country profiles in JSON for all the world's countries (using internet domain names
+for country codes e.g. Austria is `at.json` not `au.json`, Germany is `de.json` not `gm.json` and so on)
 
-## Install
 
-Just install the gem:
 
-    $ gem install factbook
+## Alternatives (Libraries and Gems)
 
+Ruby
 
-## Ready-To-Use Public Domain Datasets
+- [worldfactbook gem](https://github.com/sayem/worldfactbook)
+  by Sayem Khan (aka sayem);
+  fetches data from its own mirror, that is, [rubyworldfactbook.com](http://rubyworldfactbook.com)
+  (last updated 2011?)
 
-Datasets generated by `factbook` include:
+- [the_country_identity gem](https://github.com/p1nox/the_country_identity)
+  by Raul Pino (aka p1nox);
+  fetches data from an [RDF Turtle endpoint](http://wifo5-03.informatik.uni-mannheim.de/factbook/)
+  hosted by the Research Group Data and Web Science at the University of Mannheim, Germany
 
-[openmundi/factbook.json](https://github.com/openmundi/factbook.json) - open (public domain)
-factbook country profiles in JSON for all the world's countries (using internet domain names
-for country codes e.g. Austria is `at.json` not `au.json`, Germany is `de.json` not `gm.json` and so on)
+JavaScript
 
+- [worldfactbook-dataset](https://github.com/twigkit/worldfactbook-dataset)
+  by Richard Marr (aka richmarr); fetches data using Node.js
+  (last updated 2013)
 
+Others
 
-## Alternatives Libraries and Gems
+TBD
 
-Ruby
 
-- [worldfactbook gem](https://github.com/sayem/worldfactbook) by sayem (aka Sayem Khan); fetches data from its own mirror, that is, rubyworldfactbook.com (last updated 2011?)
 
-Others
+## Install
 
-TBD
+Just install the gem:
+
+    $ gem install factbook
 
 
 ## License
@@ -70,4 +486,4 @@ Use it as you please with no restrictions whatsoever.
 ## Questions? Comments?
 
 Send them along to the [Open Mundi (world.db) Database Forum/Mailing List](http://groups.google.com/group/openmundi).
-Thanks!
+Thanks!

data/Rakefile

@@ -32,88 +32,15 @@ Hoe.spec 'factbook' do
 end
 
 
-=begin
-# errors to fix:
-saving a copy to europe/li-liechtenstein.html for debugging
-  found section 0 @ 38
-  found section 1 @ 1882
-  found section 2 @ 13160
-  found section 3 @ 29355
-  found section 4 @ 46010
-*** error: section not found -- <div id="CollapsiblePanel1_Energy"
-  found section 6 @ 64725
-
-aving a copy to europe/mc-monaco.html for debugging
-  found section 0 @ 38
-  found section 1 @ 1446
-  found section 2 @ 12736
-  found section 3 @ 31192
-  found section 4 @ 47762
-*** error: section not found -- <div id="CollapsiblePanel1_Energy"
-
-saving a copy to europe/sm-san-marino.html for debugging
-  found section 0 @ 38
-  found section 1 @ 1379
-  found section 2 @ 12243
-  found section 3 @ 27349
-  found section 4 @ 46949
-*** error: section not found -- <div id="CollapsiblePanel1_Energy"
-
-saving a copy to europe/va-vatican-city.html for debugging
-  found section 0 @ 38
-  found section 1 @ 2000
-  found section 2 @ 13093
-  found section 3 @ 19912
-  found section 4 @ 37264
-*** error: section not found -- <div id="CollapsiblePanel1_Energy"
-  found section 6 @ 44353
-*** error: section not found -- <div id="CollapsiblePanel1_Trans"
-
-saving a copy to pacific/mh-marshall-islands.html for debugging
-  found section 0 @ 38
-  found section 1 @ 1414
-  found section 2 @ 13404
-  found section 3 @ 34854
-  found section 4 @ 52734
-*** error: section not found -- <div id="CollapsiblePanel1_Energy"
-
-saving a copy to pacific/pw-palau.html for debugging
-  found section 0 @ 38
-  found section 1 @ 1338
-  found section 2 @ 12729
-  found section 3 @ 34145
-  found section 4 @ 51005
-*** error: section not found -- <div id="CollapsiblePanel1_Energy"
-
-saving a copy to pacific/tv-tuvalu.html for debugging
-  found section 0 @ 38
-  found section 1 @ 1391
-  found section 2 @ 13580
-  found section 3 @ 33729
-  found section 4 @ 50390
-*** error: section not found -- <div id="CollapsiblePanel1_Energy"
-
-saving a copy to africa/ss-south-sudan.html for debugging
-  found section 0 @ 38
-  found section 1 @ 2560
-  found section 2 @ 11342
-  found section 3 @ 26234
-  found section 4 @ 42271
-*** error: section not found -- <div id="CollapsiblePanel1_Energy"
-
-
-=end
-
-
 
 desc 'generate json for factbook.json repo'
 task :genjson do
   require 'factbook'
 
   countries = [
-=begin
      ['xx', 'world' ],     ## special code for the world
 
+=begin
      ['ee', 'europe/eu-european-union'],  ## special code for the european union
      ['al', 'europe/al-albania' ],
      ['an', 'europe/ad-andorra' ],
@@ -140,13 +67,13 @@ task :genjson do
      ['ei', 'europe/ie-ireland' ],
      ['it', 'europe/it-italy' ],
      ['lg',   'europe/lv-latvia' ],
-#     ['ls',   'europe/li-liechtenstein' ],
+     ['ls',   'europe/li-liechtenstein' ],
      ['lh',   'europe/lt-lithuania' ],
      ['lu',   'europe/lu-luxembourg' ],
      ['mk',   'europe/mk-macedonia' ],
      ['mt',   'europe/mt-malta' ],
      ['md',   'europe/md-moldova' ],
-#     ['mn',   'europe/mc-monaco' ],
+     ['mn',   'europe/mc-monaco' ],
      ['mj',   'europe/me-montenegro' ],
      ['nl',   'europe/nl-netherlands' ],
      ['no',   'europe/no-norway' ],
@@ -154,7 +81,7 @@ task :genjson do
      ['po',   'europe/pt-portugal' ],
      ['ro',   'europe/ro-romania' ],
      ['rs',   'europe/ru-russia' ],
-#     ['sm',   'europe/sm-san-marino' ],
+     ['sm',   'europe/sm-san-marino' ],
      ['ri',   'europe/rs-serbia' ],
      ['lo',   'europe/sk-slovakia' ],
      ['si',   'europe/si-slovenia' ],
@@ -163,7 +90,7 @@ task :genjson do
      ['sz',   'europe/ch-switzerland' ],
      ['tu',   'europe/tr-turkey' ],
      ['up',   'europe/ua-ukraine' ],
-#     ['vt',   'europe/va-vatican-city' ],
+     ['vt',   'europe/va-vatican-city' ],
 
      ['ca',   'north-america/ca-canada' ],
      ['us',   'north-america/us-united-states' ],
@@ -249,7 +176,7 @@ task :genjson do
      ['sl',   'africa/sl-sierra-leone' ],
      ['so',   'africa/so-somalia' ],
      ['sf',   'africa/za-south-africa' ],
-#     ['od',   'africa/ss-south-sudan' ],
+     ['od',   'africa/ss-south-sudan' ],
      ['su',   'africa/sd-sudan' ],
      ['wz',   'africa/sz-swaziland' ],
      ['tz',   'africa/tz-tanzania' ],
@@ -308,25 +235,19 @@ task :genjson do
      ['as',   'pacific/au-australia' ],
      ['fj',   'pacific/fj-fiji' ],
      ['kr',   'pacific/ki-kiribati' ],
-#     ['rm',   'pacific/mh-marshall-islands' ],
+     ['rm',   'pacific/mh-marshall-islands' ],
      ['fm',   'pacific/fm-micronesia' ],
      ['nr',   'pacific/nr-nauru' ],
      ['nz',   'pacific/nz-new-zealand' ],
-#     ['ps',   'pacific/pw-palau' ],
+     ['ps',   'pacific/pw-palau' ],
      ['pp',   'pacific/pg-papua-new-guinea' ],
      ['ws',   'pacific/ws-samoa' ],
      ['bp',   'pacific/sb-solomon-islands' ],
      ['tn',   'pacific/to-tonga' ],
-#     ['tv',   'pacific/tv-tuvalu' ],
+     ['tv',   'pacific/tv-tuvalu' ],
      ['nh',   'pacific/vu-vanuatu' ],
-
 =end
 
-
-
-=begin
-     ['',   'africa/' ],
-=end
   ]
 
   countries.each do |country|

data/lib/factbook.rb

@@ -25,17 +25,6 @@ require 'factbook/page'
 require 'factbook/sect'
 
 
-module Factbook
-
-  def self.banner
-    "factbook/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
-  end
-
-  def self.root
-    "#{File.expand_path( File.dirname(File.dirname(__FILE__)) )}"
-  end
-
-end # module Factbook
 
 
 puts Factbook.banner

data/lib/factbook/page.rb

@@ -13,11 +13,14 @@ module Factbook
     ## e.g. www.cia.gov/library/publications/the-world-factbook/geos/countrytemplate_br.html
     SITE_BASE = 'https://www.cia.gov/library/publications/the-world-factbook/geos/countrytemplate_{code}.html'
                  
-    def initialize( code )
+    def initialize( code, opts={} )
       ## note: requires factbook country code
       #   e.g. austria is au
       #        germany is gm  and so on
       @code  = code
+      
+      ### rename fields to format option?? why? why not? e.g. :format => 'long' ??
+      @opts  = opts   # fields:  full|long|keep|std|??  -- find a good name for the option keeping field names as is
 
       @html  = nil
       @doc   = nil
@@ -38,10 +41,35 @@ module Factbook
       end
     end
 
+
+    def [](key)  ### convenience shortcut
+      # lets you use
+      #   page['geo']
+      #   instead of
+      #   page.data['geo']
+
+      ##  fix: use delegate data, [] from forwardable lib - why?? why not??
+
+      data[key]
+    end
+
+
     def data
       if @data.nil?
         @data = {}
 
+        if @opts[:header]   ## include (leading) header section ??
+          
+          header_key =     @opts[:fields] ? 'Header' : 'header'
+          last_built_key = @opts[:fields] ? 'last built' : 'last_built'
+
+          @data[header_key] = {
+            'code' => @code,
+            'generator' => "factbook/#{VERSION}",
+            last_built_key => "#{Time.now}",
+          }
+        end
+
         sects.each_with_index do |sect,i|
           logger.debug "############################"
           logger.debug "###  [#{i}] stats sect >#{sect.title}<: "
@@ -58,17 +86,18 @@ module Factbook
         ## split html into sections
         ##   lets us avoids errors w/ (wrongly) nested tags
 
+        ## check opts for using long or short category/field names
         divs = [
-          [ 'intro',    '<div id="CollapsiblePanel1_Intro"'   ],
-          [ 'geo',      '<div id="CollapsiblePanel1_Geo"'     ],
-          [ 'people',   '<div id="CollapsiblePanel1_People"'  ],
-          [ 'govt',     '<div id="CollapsiblePanel1_Govt"'    ],
-          [ 'econ',     '<div id="CollapsiblePanel1_Econ"'    ],
-          [ 'energy',   '<div id="CollapsiblePanel1_Energy"'  ],
-          [ 'comm',     '<div id="CollapsiblePanel1_Comm"'    ],
-          [ 'trans',    '<div id="CollapsiblePanel1_Trans"'   ],
-          [ 'military', '<div id="CollapsiblePanel1_Military"'],
-          [ 'issues',   '<div id="CollapsiblePanel1_Issues"'  ]
+          [ @opts[:fields] ? 'Introduction'        : 'intro',    '<div id="CollapsiblePanel1_Intro"'   ],
+          [ @opts[:fields] ? 'Geography'           : 'geo',      '<div id="CollapsiblePanel1_Geo"'     ],
+          [ @opts[:fields] ? 'People and Society'  : 'people',   '<div id="CollapsiblePanel1_People"'  ],
+          [ @opts[:fields] ? 'Government'          : 'govt',     '<div id="CollapsiblePanel1_Govt"'    ],
+          [ @opts[:fields] ? 'Economy'             : 'econ',     '<div id="CollapsiblePanel1_Econ"'    ],
+          [ @opts[:fields] ? 'Energy'              : 'energy',   '<div id="CollapsiblePanel1_Energy"'  ],
+          [ @opts[:fields] ? 'Communications'      : 'comm',     '<div id="CollapsiblePanel1_Comm"'    ],
+          [ @opts[:fields] ? 'Transportation'      : 'trans',    '<div id="CollapsiblePanel1_Trans"'   ],
+          [ @opts[:fields] ? 'Military'            : 'military', '<div id="CollapsiblePanel1_Military"'],
+          [ @opts[:fields] ? 'Transnational Issues': 'issues',   '<div id="CollapsiblePanel1_Issues"'  ]
         ]
 
         indexes = []
@@ -102,7 +131,7 @@ module Factbook
 
           ## todo: check that from is smaller than to
           logger.debug "   cut section #{i} [#{from}..#{to}]"
-          @sects << Sect.new( title, html[ from..to ] )
+          @sects << Sect.new( title, html[ from..to ], @opts )
 
           ##if i==0 || i==1
             ## puts "debug sect #{i}:"

data/lib/factbook/sect.rb

@@ -7,11 +7,12 @@ module Factbook
 
     attr_reader :title, :html
 
-    def initialize( title, html )
+    def initialize( title, html, opts={} )
       ## todo: passing a ref to the parent page - why? why not??
       @title = title
       @html  = html
-      
+      @opts  = opts    # fields:  full|long|keep|std|???  -- find a good name for the option keeping field names as is
+
       @doc   = nil
       @data  = nil
     end
@@ -28,15 +29,31 @@ module Factbook
 private
 
   def cleanup_key( key )
-    ## to lower case
-    key = key.downcase
-    ## seaport(s)  => seaports
-    key = key.gsub( '(s)', 's' )
-    key = key.gsub( ':', '' )    # trailing :
-    ## remove special chars ()-/,'
-    key = key.gsub( /['()\-\/,]/, ' ' )
-    key = key.strip
-    key = key.gsub( /[ ]+/, '_' )
+    
+    if @opts[:fields]    #  if set assume full|long|keep for now
+      ### kepe field names as is
+      ##  e.g.
+      ##   GDP - composition, by sector of origin:
+      ##   Budget surplus (+) or deficit (-):
+      ##  becomes:
+      ##   GDP - composition, by sector of origin
+      ##   Budget surplus (+) or deficit (-) 
+      key = key.strip
+      key = key.gsub( /[ ]{2,}/, ' ' )   # fold two plus spaces into one  -- check if exists?
+      key = key.gsub( /:\z/, '' )    # remove trailing : if present
+      key = key.strip
+    else
+      ## to lower case
+      key = key.downcase
+      ## seaport(s)  => seaports
+      key = key.gsub( '(s)', 's' )
+      key = key.gsub( ':', '' )    # trailing :  ## fix: use regex /:$/ w/ anchor??
+      ## remove special chars ()+-/,'
+      key = key.gsub( /['()+\-\/,]/, ' ' )
+      key = key.strip
+      key = key.gsub( /[ ]+/, '_' )
+    end
+
     key
   end
 
@@ -140,7 +157,7 @@ private
                   last_pair[1] += " #{text}"    ## append w/o separator
                 end
               else
-                if last_cat == 'demographic_profile'  ## special case (use space a sep)
+                if last_cat == 'demographic_profile' || last_cat == 'Demographic profile'  ## special case (use space a sep)
                   last_pair[1] += " #{text}"   ## append with separator
                 else
                   last_pair[1] += "; #{text}"   ## append with separator

data/lib/factbook/version.rb

@@ -1,5 +1,21 @@
 
 module Factbook
-  VERSION = '0.1.2'
-end
 
+  MAJOR = 0
+  MINOR = 1
+  PATCH = 3
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "factbook/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}"
+  end
+  
+end

data/test/test_fields.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+
+
+require 'helper'
+
+
+class TestFields < MiniTest::Unit::TestCase
+
+  def read_test_page( code )
+    File.read( "#{Factbook.root}/test/data/countrytemplate_#{code}.html" )
+  end
+
+  def test_fields_full_w_header
+    page = Factbook::Page.new( 'au', header: true, fields: 'full' )
+    page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    assert_equal 'au', page['Header']['code']
+    assert_equal "factbook/#{Factbook::VERSION}", page['Header']['generator']
+
+    assert_equal '-3.1% of GDP (2012 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
+    assert_equal '5.5%',  page['Economy']['Labor force - by occupation']['agriculture']
+
+    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']
+  end
+
+
+  def test_fields_full
+    page = Factbook::Page.new( 'au', fields: 'full' )
+    page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    assert_equal '-3.1% of GDP (2012 est.)', page['Economy']['Budget surplus (+) or deficit (-)']['text']
+    assert_equal '5.5%',  page['Economy']['Labor force - by occupation']['agriculture']
+
+    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['Transportation']['Ports and terminals']['river port(s)']
+  end
+
+  def test_fields_std
+    page = Factbook::Page.new( 'au' )
+    page.html = read_test_page( 'au' )  # use builtin test page (do NOT fetch via internet)
+
+    assert_equal '-3.1% of GDP (2012 est.)', page['econ']['budget_surplus_or_deficit']['text']
+    assert_equal '5.5%',  page['econ']['labor_force_by_occupation']['agriculture']
+
+    assert_equal 'Enns, Krems, Linz, Vienna (Danube)', page['trans']['ports_and_terminals']['river_ports']
+  end
+
+
+end # class TestFields

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: factbook
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-13 00:00:00.000000000 Z
+date: 2014-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &82513280 !ruby/object:Gem::Requirement
+  requirement: &74365370 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *82513280
+  version_requirements: *74365370
 - !ruby/object:Gem::Dependency
   name: fetcher
-  requirement: &82544000 !ruby/object:Gem::Requirement
+  requirement: &74365080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *82544000
+  version_requirements: *74365080
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &82543300 !ruby/object:Gem::Requirement
+  requirement: &74364810 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *82543300
+  version_requirements: *74364810
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &82542590 !ruby/object:Gem::Requirement
+  requirement: &74364520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,18 +54,18 @@ dependencies:
         version: '4.0'
   type: :development
   prerelease: false
-  version_requirements: *82542590
+  version_requirements: *74364520
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &82541780 !ruby/object:Gem::Requirement
+  requirement: &74364240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.11'
+        version: '3.12'
   type: :development
   prerelease: false
-  version_requirements: *82541780
+  version_requirements: *74364240
 description: factbook - scripts for the world factbook (get open structured data e.g
   JSON etc.)
 email: openmundi@googlegroups.com
@@ -93,6 +93,7 @@ files:
 - test/data/countrytemplate_vt.html
 - test/data/countrytemplate_xx.html
 - test/helper.rb
+- test/test_fields.rb
 - test/test_json.rb
 - test/test_page.rb
 - test/test_page_old.rb
@@ -129,5 +130,6 @@ summary: factbook - scripts for the world factbook (get open structured data e.g
 test_files:
 - test/test_page_old.rb
 - test/test_strip.rb
+- test/test_fields.rb
 - test/test_json.rb
 - test/test_page.rb