factbook 0.1.1 → 0.1.2

data/test/test_json.rb

@@ -6,40 +6,42 @@ require 'helper'
 
 class TestJson < MiniTest::Unit::TestCase
 
-  def setup
-    Dir.mkdir( 'tmp' )  unless Dir.exists?( 'tmp' )
-  end
 
   def test_json
-     gen_json_for( 'au')
-     gen_json_for( 'be')
-     gen_json_for( 'br')
-     gen_json_for( 'mx')
-  end
-
-  def gen_json_for( code )
-    page = Factbook::Page.new( code )    
-    page.html = File.read( "#{Factbook.root}/test/data/countrytemplate_#{code}.html" )
-
-    ## print first 600 chars
-    pp page.html[0..600]
-
-    ## save for debuging
+     Dir.mkdir( 'tmp' )  unless Dir.exists?( 'tmp' )
+     
+     codes = [ 'au',
+               'be',
+               'br',
+               'mx',
+               'ls',
+               'vt',
+               'ee',
+               'xx' ]
+
+     codes.each do |code|
+       page = Factbook::Page.new( code )    
+       page.html = File.read( "#{Factbook.root}/test/data/countrytemplate_#{code}.html" )
+
+       ## print first 600 chars
+       pp page.html[0..600]
+
+       ## save for debuging
     
-    puts "saving a copy to #{code}.html for debugging"
-    File.open( "tmp/#{code}.html", 'w') do |f|
-      f.write( page.html )
-    end
+       puts "saving a copy to #{code}.html for debugging"
+       File.open( "tmp/#{code}.html", 'w' ) do |f|
+        f.write page.html 
+       end
 
-    h = page.data
-    pp h
+       h = page.data
+       pp h
     
-    ### save to json
-    puts "saving a copy to #{code}.json for debugging"
-    File.open( "tmp/#{code}.json", 'w') do |f|
-      f.write( JSON.pretty_generate( h ) )
-    end
+       ### save to json
+       puts "saving a copy to #{code}.json for debugging"
+       File.open( "tmp/#{code}.json", 'w' ) do |f|
+        f.write JSON.pretty_generate( h )
+      end
+     end
   end
 
-
 end # class TestJson

data/test/test_page.rb

@@ -6,222 +6,31 @@ require 'helper'
 
 class TestPage < MiniTest::Unit::TestCase
 
-  def setup
-    Dir.mkdir( 'tmp' )  unless Dir.exists?( 'tmp' )
-  end
 
-  def test_br
-    page = Factbook::Page.new( 'br' )
+  def test_sects
+    pages = [
+      [ 'au', 10 ],
+      [ 'be', 10 ],
+      [ 'br', 10 ],
+      [ 'ee', 10 ],
+      [ 'mx', 10 ],
+      [ 'xx', 10 ],
+      [ 'ls', 9  ],
+      [ 'vt', 8  ]]
     
-    page.html = File.read( "#{Factbook.root}/test/data/countrytemplate_br.html" )
+    pages.each do |rec|
+        code       = rec[0]
+        sects_size = rec[1]
 
-    ## print first 600 chars
-    pp page.html[0..600]
-    
-    ## save for debuging
-    
-    puts "saving a copy to br.html for debugging"
-    File.open( 'tmp/br.html', 'w') do |f|
-      f.write( page.html )
-    end
+        page = Factbook::Page.new( code )    
+        page.html = File.read( "#{Factbook.root}/test/data/countrytemplate_#{code}.html" )
 
-    doc   = page.doc
-    sects = page.sects
+        ## print first 600 chars
+        pp page.html[0..600]
 
-    h = page.data
-    pp h
-    
-    ### save to json
-    puts "saving a copy to br.json for debugging"
-    File.open( 'tmp/br.json', 'w') do |f|
-      f.write( JSON.pretty_generate( h ) )
+        assert_equal sects_size, page.sects.size
     end
   end
 
 
-  def xxx_test_br
-    page = Factbook::Page.new( 'br' )
-    
-    ## print first 600 chars
-    pp page.html[0..600]
-    
-    ## save for debuging
-    
-    Dir.mkdir( 'tmp' )  unless Dir.exists?( 'tmp' )
-    puts "saving a copy to br.html for debugging"
-    File.open( 'tmp/br.html', 'w') do |f|
-      f.write( page.html )
-    end
-
-    doc   = page.doc
-    sects = page.sects
-
-    rows      = doc.css( 'table tr' )
-    field_ids = rows.css( '#field' )    ## check - use div#field.category -- possible?
-    data_ids  = rows.css( '#data' )
-
-    puts "rows.size:    #{rows.size}  (field_ids.size: #{field_ids.size} / data_ids.size: #{data_ids.size})"
-
-    cats      = rows.css( '.category' )
-    cats_div  = rows.css( 'div.category' )
-    cats_span = rows.css( 'span.category' )
-    cats_other_size = cats.size - cats_div.size - cats_span.size
-
-    cats_data      = rows.css( '.category_data' )
-    cats_div_data  = rows.css( 'div.category_data' )
-    cats_span_data = rows.css( 'span.category_data' )
-    cats_other_data_size = cats_data.size - cats_div_data.size - cats_span_data.size
-
-    puts "cats.size:  #{cats.size} (cats_div.size #{cats_div.size} / cats_span.size #{cats_span.size} / cats_other.size #{cats_other_size})"
-    puts "cats_data.size:  #{cats_data.size} (cats_div_data.size #{cats_div_data.size} / cats_span_data.size #{cats_span_data.size} / cats_other_data.size #{cats_other_data_size})"
-
-    ## some check for structure
-    if cats_other_size > 0
-        puts " ****!!!! category other (not div/span) found - #{cats_other_size}"
-    end
- 
-    if cats_other_data_size > 0
-        puts " ****!!!! category_data other (not div/span) found - #{cats_other_data_size}"
-    end
-
-    ## stats( doc )
-
-    sects.each_with_index do |sect,i|
-      puts ''
-      puts "############################"
-      puts "#### stats sect #{i}:"
-      pp page.sect_to_hash( sect )
-    end
-  end
-
-
-  def stats( doc )
-    rows  = doc.css( 'table tr' )
-    cells = doc.css( 'table tr td' )
-    field_ids = rows.css( '#field' )    ## check - use div#field.category -- possible?
-    data_ids  = rows.css( '#data' )
-
-    puts "rows.size:    #{rows.size}  (cells.size: #{cells.size} / field_ids.size: #{field_ids.size} / data_ids.size: #{data_ids.size})"
-
-    hash = {}
-    last_cat = nil
-
-
-    cells.each_with_index do |cell,i|
-      ## next if i > 14   ## skip after xx for debugging for now
-
-      # check if field or data id
-
-      # check for (nested) div#field in td
-      has_field_id  =  cell.css( '#field' ).size == 1 ? true : false
-
-      # check for td#data
-      has_data_id =  cell['id'] == 'data' ? true : false
-
-      if has_field_id
-        
-        cats  = cell.css( 'div.category' )   ## note: ignore all .category not using div (issue warn/err if found!!) etc.
-        if cats.size == 1
-          text = cats.first.text.strip   # remove/strip leading and trailing spaces
-          last_cat = text
-          puts "  [#{i}] category: >>#{text}<<"
-        else
-          puts "**** !!!!!! warn/err - found element w/ field id  (no match for subsection!!! - check)"
-          puts cell.to_s
-        end
-
-      elsif has_data_id
-
-        cats      = cell.css( 'div.category' )   ## note: ignore all .category not using div (issue warn/err if found!!) etc.
-        cats_data = cell.css( 'div.category_data,span.category_data' )  ## note: ignore a.category_data etc.
-        cats_div_data  =  cell.css( 'div.category_data' )
-        cats_span_data =  cell.css( 'span.category_data' )
-
-        puts "    - [#{i}] data cell - cats: #{cats.size}, cats_data: #{cats_data.size} (cats_div_data: #{cats_div_data.size} / cats_span_data: #{cats_span_data.size})"
-
-        pairs = []
-        last_pair = nil
-        last_pair_data_count = 0
-
-        ## loop over div blocks (might be .category or .category_data)
-        cell.children.each_with_index do |child,j|
-           unless child.element?
-             ## puts "   **** !!!! skipping non-element type >#{child.type}<:"
-             ## puts child.to_s
-             next
-           end
-           unless child.name == 'div'
-             puts "   **** !!! skipping non-div >#{child.name}<:"
-             puts child.to_s
-             next
-           end
-
-           ### check if .category or .category_data
-           if child['class'] == 'category'
-             
-              ## collect text for category; exclude element w/ class.category_data
-              text = ""
-              child.children.each do |subchild|
-                text << subchild.text.strip     unless subchild.element? && subchild['class'] == 'category_data'
-              end
-              
-              value = child.css('span.category_data').text.strip
-
-              puts "        -- category >>#{text}<<"
-
-              ## start new pair
-              last_pair = [ text, value ]
-              last_pair_data_count = 0
-              pairs << last_pair
-
-           elsif child['class'] == 'category_data'
-              puts "        -- category_data"
-
-              text = child.text.strip
-
-              if last_pair.nil?
-                ## assume its the very first entry; use implied/auto-created category
-                last_pair = [ 'text', '' ]
-                last_pair_data_count = 0
-                pairs << last_pair
-              end
-
-              ### first category_data element?
-              if last_pair_data_count == 0
-                if last_pair[1] == ''
-                  last_pair[1] = text
-                else
-                  last_pair[1] += " #{text}"    ## append w/o separator
-                end
-              else
-                last_pair[1] += "; #{text}"   ## append with separator 
-              end
-              last_pair_data_count += 1
-              
-           else
-              puts "  **** !!! skipping div w/o category or category_data class:"
-              puts child.to_s
-           end
-        end
-
-        ## pp pairs
-        
-        ## pairs to hash
-        pairs_hash = {}
-        pairs.each do |pair|
-          pairs_hash[ pair[0] ] = pair[1]
-        end
-
-        hash[ last_cat ] = pairs_hash
-
-      else
-        puts "#### !!!!  unknown cell type (no field or data id found):"
-        puts cell.to_s
-      end
-    end # each cell
-    
-    pp hash
-  end # method stats
-
-
 end # class TestPage

data/test/test_page_old.rb

@@ -6,7 +6,195 @@ require 'helper'
 
 class TestPageOld < MiniTest::Unit::TestCase
 
-  def xxx_test_mx
+
+  def xxx_test_br
+    Dir.mkdir( 'tmp' )  unless Dir.exists?( 'tmp' )
+
+    page = Factbook::Page.new( 'br' )
+    
+    ## print first 600 chars
+    pp page.html[0..600]
+    
+    ## save for debuging
+    
+    Dir.mkdir( 'tmp' )  unless Dir.exists?( 'tmp' )
+    puts "saving a copy to br.html for debugging"
+    File.open( 'tmp/br.html', 'w') do |f|
+      f.write( page.html )
+    end
+
+    doc   = page.doc
+    sects = page.sects
+
+    rows      = doc.css( 'table tr' )
+    field_ids = rows.css( '#field' )    ## check - use div#field.category -- possible?
+    data_ids  = rows.css( '#data' )
+
+    puts "rows.size:    #{rows.size}  (field_ids.size: #{field_ids.size} / data_ids.size: #{data_ids.size})"
+
+    cats      = rows.css( '.category' )
+    cats_div  = rows.css( 'div.category' )
+    cats_span = rows.css( 'span.category' )
+    cats_other_size = cats.size - cats_div.size - cats_span.size
+
+    cats_data      = rows.css( '.category_data' )
+    cats_div_data  = rows.css( 'div.category_data' )
+    cats_span_data = rows.css( 'span.category_data' )
+    cats_other_data_size = cats_data.size - cats_div_data.size - cats_span_data.size
+
+    puts "cats.size:  #{cats.size} (cats_div.size #{cats_div.size} / cats_span.size #{cats_span.size} / cats_other.size #{cats_other_size})"
+    puts "cats_data.size:  #{cats_data.size} (cats_div_data.size #{cats_div_data.size} / cats_span_data.size #{cats_span_data.size} / cats_other_data.size #{cats_other_data_size})"
+
+    ## some check for structure
+    if cats_other_size > 0
+        puts " ****!!!! category other (not div/span) found - #{cats_other_size}"
+    end
+ 
+    if cats_other_data_size > 0
+        puts " ****!!!! category_data other (not div/span) found - #{cats_other_data_size}"
+    end
+
+    ## stats( doc )
+
+    sects.each_with_index do |sect,i|
+      puts ''
+      puts "############################"
+      puts "#### stats sect #{i}:"
+      pp page.sect_to_hash( sect )
+    end
+  end
+
+
+  def xxx_stats( doc )
+    rows  = doc.css( 'table tr' )
+    cells = doc.css( 'table tr td' )
+    field_ids = rows.css( '#field' )    ## check - use div#field.category -- possible?
+    data_ids  = rows.css( '#data' )
+
+    puts "rows.size:    #{rows.size}  (cells.size: #{cells.size} / field_ids.size: #{field_ids.size} / data_ids.size: #{data_ids.size})"
+
+    hash = {}
+    last_cat = nil
+
+
+    cells.each_with_index do |cell,i|
+      ## next if i > 14   ## skip after xx for debugging for now
+
+      # check if field or data id
+
+      # check for (nested) div#field in td
+      has_field_id  =  cell.css( '#field' ).size == 1 ? true : false
+
+      # check for td#data
+      has_data_id =  cell['id'] == 'data' ? true : false
+
+      if has_field_id
+        
+        cats  = cell.css( 'div.category' )   ## note: ignore all .category not using div (issue warn/err if found!!) etc.
+        if cats.size == 1
+          text = cats.first.text.strip   # remove/strip leading and trailing spaces
+          last_cat = text
+          puts "  [#{i}] category: >>#{text}<<"
+        else
+          puts "**** !!!!!! warn/err - found element w/ field id  (no match for subsection!!! - check)"
+          puts cell.to_s
+        end
+
+      elsif has_data_id
+
+        cats      = cell.css( 'div.category' )   ## note: ignore all .category not using div (issue warn/err if found!!) etc.
+        cats_data = cell.css( 'div.category_data,span.category_data' )  ## note: ignore a.category_data etc.
+        cats_div_data  =  cell.css( 'div.category_data' )
+        cats_span_data =  cell.css( 'span.category_data' )
+
+        puts "    - [#{i}] data cell - cats: #{cats.size}, cats_data: #{cats_data.size} (cats_div_data: #{cats_div_data.size} / cats_span_data: #{cats_span_data.size})"
+
+        pairs = []
+        last_pair = nil
+        last_pair_data_count = 0
+
+        ## loop over div blocks (might be .category or .category_data)
+        cell.children.each_with_index do |child,j|
+           unless child.element?
+             ## puts "   **** !!!! skipping non-element type >#{child.type}<:"
+             ## puts child.to_s
+             next
+           end
+           unless child.name == 'div'
+             puts "   **** !!! skipping non-div >#{child.name}<:"
+             puts child.to_s
+             next
+           end
+
+           ### check if .category or .category_data
+           if child['class'] == 'category'
+             
+              ## collect text for category; exclude element w/ class.category_data
+              text = ""
+              child.children.each do |subchild|
+                text << subchild.text.strip     unless subchild.element? && subchild['class'] == 'category_data'
+              end
+              
+              value = child.css('span.category_data').text.strip
+
+              puts "        -- category >>#{text}<<"
+
+              ## start new pair
+              last_pair = [ text, value ]
+              last_pair_data_count = 0
+              pairs << last_pair
+
+           elsif child['class'] == 'category_data'
+              puts "        -- category_data"
+
+              text = child.text.strip
+
+              if last_pair.nil?
+                ## assume its the very first entry; use implied/auto-created category
+                last_pair = [ 'text', '' ]
+                last_pair_data_count = 0
+                pairs << last_pair
+              end
+
+              ### first category_data element?
+              if last_pair_data_count == 0
+                if last_pair[1] == ''
+                  last_pair[1] = text
+                else
+                  last_pair[1] += " #{text}"    ## append w/o separator
+                end
+              else
+                last_pair[1] += "; #{text}"   ## append with separator 
+              end
+              last_pair_data_count += 1
+              
+           else
+              puts "  **** !!! skipping div w/o category or category_data class:"
+              puts child.to_s
+           end
+        end
+
+        ## pp pairs
+        
+        ## pairs to hash
+        pairs_hash = {}
+        pairs.each do |pair|
+          pairs_hash[ pair[0] ] = pair[1]
+        end
+
+        hash[ last_cat ] = pairs_hash
+
+      else
+        puts "#### !!!!  unknown cell type (no field or data id found):"
+        puts cell.to_s
+      end
+    end # each cell
+    
+    pp hash
+  end # method stats
+
+
+  def yyy_test_mx
     page = Factbook::Page.new( 'mx' )
     
     ## print first 600 chars
@@ -74,7 +262,7 @@ class TestPageOld < MiniTest::Unit::TestCase
 
   end
 
-  def xxxx_test_mx
+  def yyy_test_mx
     page = Factbook::Page.new( 'mx' )
     
     ## print first 600 chars
@@ -147,7 +335,7 @@ class TestPageOld < MiniTest::Unit::TestCase
   end
 
 
-  def stats( doc )
+  def yyy_stats( doc )
     rows  = doc.css( 'table tr' )
     cells = doc.css( 'table tr td' )
     field_ids = rows.css( '#field' )    ## check - use div#field.category -- possible?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: factbook
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-12 00:00:00.000000000 Z
+date: 2014-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &73732120 !ruby/object:Gem::Requirement
+  requirement: &82513280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *73732120
+  version_requirements: *82513280
 - !ruby/object:Gem::Dependency
   name: fetcher
-  requirement: &73731730 !ruby/object:Gem::Requirement
+  requirement: &82544000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *73731730
+  version_requirements: *82544000
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &73731340 !ruby/object:Gem::Requirement
+  requirement: &82543300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *73731340
+  version_requirements: *82543300
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &73730940 !ruby/object:Gem::Requirement
+  requirement: &82542590 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '4.0'
   type: :development
   prerelease: false
-  version_requirements: *73730940
+  version_requirements: *82542590
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &73730530 !ruby/object:Gem::Requirement
+  requirement: &82541780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '3.11'
   type: :development
   prerelease: false
-  version_requirements: *73730530
+  version_requirements: *82541780
 description: factbook - scripts for the world factbook (get open structured data e.g
   JSON etc.)
 email: openmundi@googlegroups.com
@@ -82,11 +82,16 @@ files:
 - Rakefile
 - lib/factbook.rb
 - lib/factbook/page.rb
+- lib/factbook/sect.rb
 - lib/factbook/version.rb
 - test/data/countrytemplate_au.html
 - test/data/countrytemplate_be.html
 - test/data/countrytemplate_br.html
+- test/data/countrytemplate_ee.html
+- test/data/countrytemplate_ls.html
 - test/data/countrytemplate_mx.html
+- test/data/countrytemplate_vt.html
+- test/data/countrytemplate_xx.html
 - test/helper.rb
 - test/test_json.rb
 - test/test_page.rb