factbook 2.0.0 → 2.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1 -1
- data/Manifest.txt +0 -61
- data/README.md +8 -506
- data/Rakefile +4 -9
- data/lib/factbook.rb +4 -64
- metadata +6 -124
- data/data/attributes.yml +0 -337
- data/data/categories.csv +0 -164
- data/data/codes.csv +0 -262
- data/data/codesxref.csv +0 -280
- data/data/comparisons.csv +0 -75
- data/lib/factbook/almanac.rb +0 -72
- data/lib/factbook/attributes.rb +0 -74
- data/lib/factbook/builder.rb +0 -212
- data/lib/factbook/builder_item.rb +0 -126
- data/lib/factbook/builder_json.rb +0 -79
- data/lib/factbook/codes.rb +0 -119
- data/lib/factbook/comparisons.rb +0 -50
- data/lib/factbook/counter.rb +0 -48
- data/lib/factbook/db/importer.rb +0 -92
- data/lib/factbook/db/models.rb +0 -11
- data/lib/factbook/db/schema.rb +0 -36
- data/lib/factbook/normalize.rb +0 -43
- data/lib/factbook/page.rb +0 -148
- data/lib/factbook/page_info.rb +0 -12
- data/lib/factbook/reader_json.rb +0 -51
- data/lib/factbook/sanitizer.rb +0 -178
- data/lib/factbook/sect.rb +0 -29
- data/lib/factbook/subsect.rb +0 -18
- data/lib/factbook/table.rb +0 -52
- data/lib/factbook/utils.rb +0 -85
- data/lib/factbook/utils_info.rb +0 -129
- data/lib/factbook/version.rb +0 -21
- data/script/almanac.rb +0 -48
- data/script/attributes.rb +0 -34
- data/script/build.rb +0 -28
- data/script/counter.rb +0 -145
- data/script/json.rb +0 -19
- data/script/testbr.rb +0 -33
- data/script/testcodes.rb +0 -11
- data/test/data/au.html +0 -579
- data/test/data/au.yml +0 -8
- data/test/data/be.html +0 -596
- data/test/data/be.yml +0 -8
- data/test/data/json/au.json +0 -892
- data/test/data/src/ag.html +0 -716
- data/test/data/src/au-2015-09-24.html +0 -2006
- data/test/data/src/au.html +0 -658
- data/test/data/src/be-2015-09-24.html +0 -2011
- data/test/data/src/be.html +0 -648
- data/test/helper.rb +0 -11
- data/test/test_attribs.rb +0 -87
- data/test/test_attribs_def.rb +0 -20
- data/test/test_builder.rb +0 -35
- data/test/test_codes.rb +0 -76
- data/test/test_comparisons.rb +0 -19
- data/test/test_convert.rb +0 -30
- data/test/test_counter.rb +0 -31
- data/test/test_fields.rb +0 -52
- data/test/test_importer.rb +0 -56
- data/test/test_item_builder.rb +0 -99
- data/test/test_json.rb +0 -45
- data/test/test_json_builder.rb +0 -25
- data/test/test_normalize.rb +0 -23
- data/test/test_page.rb +0 -38
- data/test/test_sanitizer.rb +0 -39
- data/test/test_sanitizer_regex.rb +0 -89
data/test/test_json.rb
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
###
|
|
4
|
-
# to run use
|
|
5
|
-
# ruby -I ./lib -I ./test test/test_json.rb
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
require 'helper'
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class TestJson < MiniTest::Test
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def test_json
|
|
15
|
-
Dir.mkdir( 'tmp' ) unless Dir.exists?( 'tmp' )
|
|
16
|
-
|
|
17
|
-
codes = [ 'au',
|
|
18
|
-
'ag',
|
|
19
|
-
'be',
|
|
20
|
-
#'br',
|
|
21
|
-
#'mx',
|
|
22
|
-
#'ls',
|
|
23
|
-
#'vt',
|
|
24
|
-
#'ee',
|
|
25
|
-
#'xx'
|
|
26
|
-
]
|
|
27
|
-
|
|
28
|
-
codes.each do |code|
|
|
29
|
-
|
|
30
|
-
html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
|
|
31
|
-
page = Factbook::Page.new( code, html: html )
|
|
32
|
-
|
|
33
|
-
h = page.data
|
|
34
|
-
pp h
|
|
35
|
-
|
|
36
|
-
### save to json
|
|
37
|
-
puts "saving a copy to #{code}.json for debugging"
|
|
38
|
-
File.open( "tmp/#{code}.json", 'w' ) do |f|
|
|
39
|
-
f.write JSON.pretty_generate( h )
|
|
40
|
-
## f.write page.to_json
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
end # class TestJson
|
data/test/test_json_builder.rb
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
###
|
|
4
|
-
# to run use
|
|
5
|
-
# ruby -I ./lib -I ./test test/test_json_builder.rb
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
require 'helper'
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class TestJsonBuilder < MiniTest::Test
|
|
12
|
-
|
|
13
|
-
def test_read
|
|
14
|
-
code = 'au'
|
|
15
|
-
b = Factbook::JsonBuilder.from_file( "#{Factbook.root}/test/data/json/#{code}.json" )
|
|
16
|
-
|
|
17
|
-
assert_equal 10, b.sects.size
|
|
18
|
-
assert_equal 1, b.sects[0].subsects.size ## e.g. Introduction/Background
|
|
19
|
-
assert_equal 'Central Europe, north of Italy and Slovenia', b.json['Geography']['Location']['text']
|
|
20
|
-
|
|
21
|
-
assert true ## assume everthing ok
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
end # class TestJsonBuilder
|
|
25
|
-
|
data/test/test_normalize.rb
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
###
|
|
4
|
-
# to run use
|
|
5
|
-
# ruby -I ./lib -I ./test test/test_normalize.rb
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
require 'helper'
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class TestNormalizer < MiniTest::Test
|
|
12
|
-
|
|
13
|
-
include Factbook::NormalizeHelper
|
|
14
|
-
|
|
15
|
-
def test_normalize
|
|
16
|
-
assert_equal 'border countries', normalize_category( 'border countries:' )
|
|
17
|
-
assert_equal 'border countries', normalize_category( 'border countries: ' )
|
|
18
|
-
assert_equal 'border countries', normalize_category( 'border countries (8):' )
|
|
19
|
-
assert_equal 'border countries', normalize_category( 'border countries (10): ' )
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
end # class TestNormalizer
|
|
23
|
-
|
data/test/test_page.rb
DELETED
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
###
|
|
4
|
-
# to run use
|
|
5
|
-
# ruby -I ./lib -I ./test test/test_page.rb
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
require 'helper'
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class TestPage < MiniTest::Test
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def test_sects
|
|
15
|
-
pages = [
|
|
16
|
-
[ 'au', 10 ],
|
|
17
|
-
[ 'be', 10 ],
|
|
18
|
-
# [ 'br', 10 ],
|
|
19
|
-
# [ 'ee', 10 ],
|
|
20
|
-
# [ 'mx', 10 ],
|
|
21
|
-
# [ 'xx', 10 ],
|
|
22
|
-
# [ 'ls', 9 ],
|
|
23
|
-
# [ 'vt', 8 ],
|
|
24
|
-
]
|
|
25
|
-
|
|
26
|
-
pages.each do |rec|
|
|
27
|
-
code = rec[0]
|
|
28
|
-
sects_size = rec[1]
|
|
29
|
-
|
|
30
|
-
html = File.read( "#{Factbook.root}/test/data/src/#{code}.html" )
|
|
31
|
-
page = Factbook::Page.new( code, html: html )
|
|
32
|
-
|
|
33
|
-
assert_equal sects_size, page.sects.size
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
end # class TestPage
|
data/test/test_sanitizer.rb
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
###
|
|
4
|
-
# to run use
|
|
5
|
-
# ruby -I ./lib -I ./test test/test_sanitizer.rb
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
require 'helper'
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class TestSanitizer < MiniTest::Test
|
|
12
|
-
|
|
13
|
-
def test_sanitize
|
|
14
|
-
|
|
15
|
-
## austria (au)
|
|
16
|
-
## algeria (ag)
|
|
17
|
-
## belgium (be)
|
|
18
|
-
## ['au'].each do |cnty|
|
|
19
|
-
['au','ag','be'].each do |cnty|
|
|
20
|
-
|
|
21
|
-
## use/fix: ASCII-8BIT (e.g.keep as is) -???
|
|
22
|
-
html_ascii = File.read( "#{Factbook.root}/test/data/src/#{cnty}.html" ) ## fix/todo: use ASCII8BIT/binary reader ??
|
|
23
|
-
|
|
24
|
-
html, info, errors = Factbook::Sanitizer.new.sanitize( html_ascii )
|
|
25
|
-
|
|
26
|
-
File.open( "./tmp/#{cnty}.profile.html", 'w' ) do |f|
|
|
27
|
-
f.write "** info:\n"
|
|
28
|
-
f.write info.inspect + "\n\n"
|
|
29
|
-
f.write "** errors:\n"
|
|
30
|
-
f.write errors.inspect + "\n\n"
|
|
31
|
-
f.write "** html:\n"
|
|
32
|
-
f.write html
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
assert true ## assume everthing ok
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
end # class TestSanitizer
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
###
|
|
4
|
-
# to run use
|
|
5
|
-
# ruby -I ./lib -I ./test test/test_sanitizer_regex.rb
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
require 'helper'
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class TestSanitizerRegex < MiniTest::Test
|
|
12
|
-
|
|
13
|
-
def test_area_map
|
|
14
|
-
|
|
15
|
-
html =<<HTML
|
|
16
|
-
<div class='disTable areaComp'>
|
|
17
|
-
<span class='category tCell' style='margin-bottom:0px; vertical-align:bottom;'>Area comparison map:</span>
|
|
18
|
-
<span class="tCell"><a data-toggle="modal" href="#areaCompModal"><img src="../graphics/areacomparison_icon.jpg" border="0" style="cursor:pointer; border: 0px solid #CCC;"></a></span></div>
|
|
19
|
-
|
|
20
|
-
<div class="modal fade" id="areaCompModal" role="dialog">
|
|
21
|
-
<div class="wfb-modal-dialog">
|
|
22
|
-
<div class="modal-content" >
|
|
23
|
-
<div class="wfb-modal-header" style="border-radius: 4px; font-family: Verdana,Arial,sans-serif; font-size: 14px !important; font-weight: bold; padding: 0.4em 16px 0.4em 1em; background: #cccccc url("..images/ui-bg_highlight-soft_75_cccccc_1x100.png") repeat-x scroll 50% 50%;" >
|
|
24
|
-
<span style="font-size: 14px !important; margin: 0.1em 16px 0.1em 0;" class="modal-title wfb-title">The World Factbook</span><span style="float: right; margin-top: -4px;">
|
|
25
|
-
<button type="button" class="close" title="close" data-dismiss="modal">×</button></span>
|
|
26
|
-
</div>
|
|
27
|
-
<div class="wfb-modal-body">
|
|
28
|
-
...
|
|
29
|
-
<div id='field'
|
|
30
|
-
HTML
|
|
31
|
-
|
|
32
|
-
m = Factbook::Sanitizer::AREA_COMP_CATEGORY_REGEX.match( html )
|
|
33
|
-
pp m
|
|
34
|
-
|
|
35
|
-
assert m.nil? == false
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def test_pop_pyramid
|
|
40
|
-
|
|
41
|
-
html =<<HTML
|
|
42
|
-
<div class='disTable popPyramid'>
|
|
43
|
-
<span class='category tCell' style='margin-bottom:0px; vertical-align:bottom;'>population pyramid:</span>
|
|
44
|
-
<span class="tCell"><a data-toggle="modal" href="#popPyramidModal"><img title="" src="../graphics/poppyramid_icon.jpg" style="cursor:pointer; border: 0px solid #CCC;"></span></a></div>
|
|
45
|
-
|
|
46
|
-
<div class="modal fade" id="popPyramidModal" role="dialog">
|
|
47
|
-
<div class="wfb-modal-dialog">
|
|
48
|
-
<div class="modal-content" >
|
|
49
|
-
<div class="wfb-modal-header" style="border-radius: 4px; font-family: Verdana,Arial,sans-serif; font-size: 14px !important; font-weight: bold; padding: 0.4em 16px 0.4em 1em; background: #cccccc url("..images/ui-bg_highlight-soft_75_cccccc_1x100.png") repeat-x scroll 50% 50%;" >
|
|
50
|
-
<span style="font-size: 14px !important; margin: 0.1em 16px 0.1em 0;" class="modal-title wfb-title">The World Factbook</span><span style="float: right; margin-top: -4px;">
|
|
51
|
-
<button type="button" class="close" title="close" data-dismiss="modal">×</button></span>
|
|
52
|
-
</div>
|
|
53
|
-
<div class="wfb-modal-body">
|
|
54
|
-
...
|
|
55
|
-
<div id='field'
|
|
56
|
-
HTML
|
|
57
|
-
|
|
58
|
-
m = Factbook::Sanitizer::POP_PYRAMID_CATEGORY_REGEX.match( html )
|
|
59
|
-
pp m
|
|
60
|
-
|
|
61
|
-
assert m.nil? == false
|
|
62
|
-
end # method test_pop_pyramid
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def test_rel_affiliation
|
|
66
|
-
|
|
67
|
-
html =<<HTML
|
|
68
|
-
<div class='disTable relAffiliation'><span class='category tCell' style='margin-bottom:0px; vertical-align:bottom;'>religious affiliation:</span>
|
|
69
|
-
<span class="tCell"><a data-toggle="modal" href="#relAffiliationModal"><img title="" src="../graphics/middle-east-religion-icon.jpg" style="cursor:pointer; border: 0px solid #CCC;"></span></a></div>
|
|
70
|
-
|
|
71
|
-
<div class="modal fade" id="relAffiliationModal" role="dialog">
|
|
72
|
-
<div class="wfb-modal-dialog">
|
|
73
|
-
<div class="modal-content" >
|
|
74
|
-
<div class="wfb-modal-header" style="border-radius: 4px; font-family: Verdana,Arial,sans-serif; font-size: 14px !important; font-weight: bold; padding: 0.4em 16px 0.4em 1em; background: #cccccc url("..images/ui-bg_highlight-soft_75_cccccc_1x100.png") repeat-x scroll 50% 50%;" >
|
|
75
|
-
<span style="font-size: 14px !important; margin: 0.1em 16px 0.1em 0;" class="modal-title wfb-title">The World Factbook</span><span style="float: right; margin-top: -4px;">
|
|
76
|
-
<button type="button" class="close" title="close" data-dismiss="modal">×</button></span>
|
|
77
|
-
</div>
|
|
78
|
-
<div class="wfb-modal-body">
|
|
79
|
-
...
|
|
80
|
-
<div id='field'
|
|
81
|
-
HTML
|
|
82
|
-
|
|
83
|
-
m = Factbook::Sanitizer::REL_AFFILIATION_CATEGORY_REGEX.match( html )
|
|
84
|
-
pp m
|
|
85
|
-
|
|
86
|
-
assert m.nil? == false
|
|
87
|
-
end # method test_rel_affiliation
|
|
88
|
-
|
|
89
|
-
end # class TestSanitizerRegex
|