world_bank_fetcher 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +17 -0
- data/.rvmrc +1 -0
- data/Gemfile +6 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +21 -0
- data/country_list.txt +214 -0
- data/lib/world_bank_fetcher/country_parser.rb +17 -0
- data/lib/world_bank_fetcher/data_parser.rb +16 -0
- data/lib/world_bank_fetcher/job.rb +54 -0
- data/lib/world_bank_fetcher/query_scheduler.rb +56 -0
- data/lib/world_bank_fetcher/version.rb +3 -0
- data/lib/world_bank_fetcher.rb +11 -0
- data/non_country_list.txt +32 -0
- data/sorted_countries.txt +0 -0
- data/spec/country_parser_spec.rb +33 -0
- data/spec/data_parser_spec.rb +47 -0
- data/spec/helper.rb +2 -0
- data/spec/job_spec.rb +77 -0
- data/spec/query_scheduler_spec.rb +82 -0
- data/world_bank_fetcher.gemspec +20 -0
- metadata +103 -0
data/.gitignore
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm use 1.9.2@world_bank_fetcher
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Alexander Auritt
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# WorldBankFetcher
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'world_bank_fetcher'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install world_bank_fetcher
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require 'rspec/core/rake_task'
|
4
|
+
require './lib/world_bank_fetcher'
|
5
|
+
|
6
|
+
RSpec::Core::RakeTask.new(:spec)
|
7
|
+
# 'SP.POP.TOTL'
|
8
|
+
task :test => :spec
|
9
|
+
task :default => :spec
|
10
|
+
|
11
|
+
task :indicator, :indicator_code do |t, args|
|
12
|
+
# puts "code is #{args[:indicator_code]}"
|
13
|
+
puts WorldBankFetcher::Job.new(:indicator => args[:indicator_code]).fetch ? "QUERY SUCCESSFUL!" : "QUERY FAILURE!!!!!!!!!!"
|
14
|
+
end
|
15
|
+
|
16
|
+
task :countries do
|
17
|
+
results = WorldBankFetcher::Job.new(:countries => true).fetch
|
18
|
+
puts results ? "QUERY SUCCESSFUL!" : "QUERY FAILURE!!!!!!!!!!"
|
19
|
+
# puts "here are results"
|
20
|
+
puts "#{results}"
|
21
|
+
end
|
data/country_list.txt
ADDED
@@ -0,0 +1,214 @@
|
|
1
|
+
Aruba
|
2
|
+
Afghanistan
|
3
|
+
Angola
|
4
|
+
Albania
|
5
|
+
Andorra
|
6
|
+
United Arab Emirates
|
7
|
+
Argentina
|
8
|
+
Armenia
|
9
|
+
American Samoa
|
10
|
+
Antigua and Barbuda
|
11
|
+
Australia
|
12
|
+
Austria
|
13
|
+
Azerbaijan
|
14
|
+
Burundi
|
15
|
+
Belgium
|
16
|
+
Benin
|
17
|
+
Burkina Faso
|
18
|
+
Bangladesh
|
19
|
+
Bulgaria
|
20
|
+
Bahrain
|
21
|
+
Bahamas, The
|
22
|
+
Bosnia and Herzegovina
|
23
|
+
Belarus
|
24
|
+
Belize
|
25
|
+
Bermuda
|
26
|
+
Bolivia
|
27
|
+
Brazil
|
28
|
+
Barbados
|
29
|
+
Brunei Darussalam
|
30
|
+
Bhutan
|
31
|
+
Channel Islands
|
32
|
+
Hong Kong SAR, China
|
33
|
+
Botswana
|
34
|
+
Israel
|
35
|
+
Macao SAR, China
|
36
|
+
West Bank and Gaza
|
37
|
+
Central African Republic
|
38
|
+
Canada
|
39
|
+
Switzerland
|
40
|
+
Chile
|
41
|
+
China
|
42
|
+
Cote d'Ivoire
|
43
|
+
Cameroon
|
44
|
+
Congo, Dem. Rep.
|
45
|
+
Congo, Rep.
|
46
|
+
Colombia
|
47
|
+
Comoros
|
48
|
+
Cape Verde
|
49
|
+
Costa Rica
|
50
|
+
Cuba
|
51
|
+
Curacao
|
52
|
+
Cayman Islands
|
53
|
+
Cyprus
|
54
|
+
Czech Republic
|
55
|
+
Germany
|
56
|
+
Djibouti
|
57
|
+
Dominica
|
58
|
+
Denmark
|
59
|
+
Dominican Republic
|
60
|
+
Algeria
|
61
|
+
Ecuador
|
62
|
+
Egypt, Arab Rep.
|
63
|
+
Eritrea
|
64
|
+
Spain
|
65
|
+
Estonia
|
66
|
+
Ethiopia
|
67
|
+
Finland
|
68
|
+
Fiji
|
69
|
+
France
|
70
|
+
Faeroe Islands
|
71
|
+
Micronesia, Fed. Sts.
|
72
|
+
Gabon
|
73
|
+
United Kingdom
|
74
|
+
Georgia
|
75
|
+
Ghana
|
76
|
+
Guinea
|
77
|
+
Gambia, The
|
78
|
+
Guinea-Bissau
|
79
|
+
Equatorial Guinea
|
80
|
+
Greece
|
81
|
+
Grenada
|
82
|
+
Greenland
|
83
|
+
Guatemala
|
84
|
+
Guam
|
85
|
+
Guyana
|
86
|
+
Honduras
|
87
|
+
Croatia
|
88
|
+
Haiti
|
89
|
+
Hungary
|
90
|
+
Indonesia
|
91
|
+
Isle of Man
|
92
|
+
India
|
93
|
+
Ireland
|
94
|
+
Iran, Islamic Rep.
|
95
|
+
Iraq
|
96
|
+
Iceland
|
97
|
+
Italy
|
98
|
+
Jamaica
|
99
|
+
Jordan
|
100
|
+
Japan
|
101
|
+
Kazakhstan
|
102
|
+
Kenya
|
103
|
+
Kyrgyz Republic
|
104
|
+
Cambodia
|
105
|
+
Kiribati
|
106
|
+
St. Kitts and Nevis
|
107
|
+
Korea, Rep.
|
108
|
+
Kosovo
|
109
|
+
Kuwait
|
110
|
+
Lao PDR
|
111
|
+
Lebanon
|
112
|
+
Liberia
|
113
|
+
Libya
|
114
|
+
St. Lucia
|
115
|
+
Liechtenstein
|
116
|
+
Sri Lanka
|
117
|
+
Lesotho
|
118
|
+
Lithuania
|
119
|
+
Luxembourg
|
120
|
+
Latvia
|
121
|
+
St. Martin (French part)
|
122
|
+
Morocco
|
123
|
+
Monaco
|
124
|
+
Moldova
|
125
|
+
Madagascar
|
126
|
+
Maldives
|
127
|
+
Mexico
|
128
|
+
Marshall Islands
|
129
|
+
Macedonia, FYR
|
130
|
+
Mali
|
131
|
+
Malta
|
132
|
+
Myanmar
|
133
|
+
Montenegro
|
134
|
+
Mongolia
|
135
|
+
Northern Mariana Islands
|
136
|
+
Mozambique
|
137
|
+
Mauritania
|
138
|
+
Mauritius
|
139
|
+
Malawi
|
140
|
+
Malaysia
|
141
|
+
Namibia
|
142
|
+
New Caledonia
|
143
|
+
Niger
|
144
|
+
Nigeria
|
145
|
+
Nicaragua
|
146
|
+
Netherlands
|
147
|
+
Norway
|
148
|
+
Nepal
|
149
|
+
New Zealand
|
150
|
+
Oman
|
151
|
+
Pakistan
|
152
|
+
Panama
|
153
|
+
Peru
|
154
|
+
Philippines
|
155
|
+
Palau
|
156
|
+
Papua New Guinea
|
157
|
+
Poland
|
158
|
+
Puerto Rico
|
159
|
+
Korea, Dem. Rep.
|
160
|
+
Portugal
|
161
|
+
Paraguay
|
162
|
+
French Polynesia
|
163
|
+
Qatar
|
164
|
+
Romania
|
165
|
+
Russian Federation
|
166
|
+
Rwanda
|
167
|
+
Saudi Arabia
|
168
|
+
Sudan
|
169
|
+
Senegal
|
170
|
+
Singapore
|
171
|
+
Solomon Islands
|
172
|
+
Sierra Leone
|
173
|
+
El Salvador
|
174
|
+
San Marino
|
175
|
+
Somalia
|
176
|
+
Serbia
|
177
|
+
South Sudan
|
178
|
+
Sao Tome and Principe
|
179
|
+
Suriname
|
180
|
+
Slovak Republic
|
181
|
+
Slovenia
|
182
|
+
Sweden
|
183
|
+
Swaziland
|
184
|
+
Sint Maarten (Dutch part)
|
185
|
+
Seychelles
|
186
|
+
Syrian Arab Republic
|
187
|
+
Turks and Caicos Islands
|
188
|
+
Chad
|
189
|
+
Togo
|
190
|
+
Thailand
|
191
|
+
Tajikistan
|
192
|
+
Turkmenistan
|
193
|
+
Timor-Leste
|
194
|
+
Tonga
|
195
|
+
Trinidad and Tobago
|
196
|
+
Tunisia
|
197
|
+
Turkey
|
198
|
+
Tuvalu
|
199
|
+
Tanzania
|
200
|
+
Uganda
|
201
|
+
Ukraine
|
202
|
+
Uruguay
|
203
|
+
United States
|
204
|
+
Uzbekistan
|
205
|
+
St. Vincent and the Grenadines
|
206
|
+
Venezuela, RB
|
207
|
+
Virgin Islands (U.S.)
|
208
|
+
Vietnam
|
209
|
+
Vanuatu
|
210
|
+
Samoa
|
211
|
+
Yemen, Rep.
|
212
|
+
South Africa
|
213
|
+
Zambia
|
214
|
+
Zimbabwe
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'world_bank'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
class CountryParser
|
5
|
+
def self.parse(data_collection)
|
6
|
+
results = []
|
7
|
+
data_collection.each do |datum|
|
8
|
+
current_country = {}
|
9
|
+
current_country[:name] = datum.name
|
10
|
+
current_country[:capital] = datum.capital
|
11
|
+
current_country[:region] = datum.region.raw.value
|
12
|
+
results << current_country
|
13
|
+
end
|
14
|
+
results
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'world_bank'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
class DataParser
|
5
|
+
def self.parse(data_collection)
|
6
|
+
indicator_id = data_collection.first.id
|
7
|
+
results = {:id => indicator_id }
|
8
|
+
data_collection.each do |datum|
|
9
|
+
return nil unless datum.id === indicator_id
|
10
|
+
results[datum.others['country']['value']] ||= []
|
11
|
+
results[datum.others['country']['value']] << { :year => datum.date, :value => datum.value }
|
12
|
+
end
|
13
|
+
results
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'world_bank'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
class Job
|
5
|
+
attr_reader :results, :checksum
|
6
|
+
|
7
|
+
def initialize(options)
|
8
|
+
@results = nil
|
9
|
+
@checksum = nil
|
10
|
+
@job_type = options[:indicator] ? :indicator : :country
|
11
|
+
@query = build_query options
|
12
|
+
end
|
13
|
+
|
14
|
+
def fetch
|
15
|
+
data = fetch_all_data query
|
16
|
+
if data
|
17
|
+
if @job_type == :indicator
|
18
|
+
@results = WorldBankFetcher::DataParser.parse data
|
19
|
+
else
|
20
|
+
@results = WordBankFetcher::CountryParser.parse data
|
21
|
+
end
|
22
|
+
|
23
|
+
@checksum = Digest::MD5.hexdigest Marshal.dump(@results)
|
24
|
+
{:results => @results, :checksum => @checksum}
|
25
|
+
else
|
26
|
+
nil
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def build_query(options)
|
33
|
+
options[:indicator] ? indicator_query(options[:indicator]) : countries_query
|
34
|
+
end
|
35
|
+
|
36
|
+
def indicator_query(indicator_code)
|
37
|
+
WorldBank::Data.country('all').indicator(indicator_code)
|
38
|
+
end
|
39
|
+
|
40
|
+
def countries_query
|
41
|
+
WorldBank::Country.all
|
42
|
+
end
|
43
|
+
|
44
|
+
def query
|
45
|
+
@query
|
46
|
+
end
|
47
|
+
|
48
|
+
def fetch_all_data(query)
|
49
|
+
scheduler = WorldBankFetcher::QueryScheduler.new query
|
50
|
+
query.per_page(WorldBankFetcher::MAXIMUM_BUFFER_SIZE)
|
51
|
+
scheduler.execute!
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module WorldBankFetcher
|
2
|
+
class QueryScheduler
|
3
|
+
attr_reader :query
|
4
|
+
|
5
|
+
def initialize(query)
|
6
|
+
@query = query
|
7
|
+
@results = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def execute!
|
11
|
+
query.per_page(WorldBankFetcher::MAXIMUM_BUFFER_SIZE)
|
12
|
+
begin
|
13
|
+
if query.is_a? WorldBank::ParamQuery
|
14
|
+
fetch_param_query!
|
15
|
+
elsif query.is_a? WorldBank::DataQuery
|
16
|
+
fetch_data_query!
|
17
|
+
else
|
18
|
+
raise
|
19
|
+
end
|
20
|
+
rescue
|
21
|
+
nil
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def fetch_param_query!
|
28
|
+
results = query.fetch
|
29
|
+
rails if results.size === MAXIMUM_BUFFER_SIZE
|
30
|
+
results
|
31
|
+
end
|
32
|
+
|
33
|
+
def fetch_data_query!
|
34
|
+
results = []
|
35
|
+
total_queries.times do |i|
|
36
|
+
res = query.page(i+1).fetch
|
37
|
+
return nil if res.nil?
|
38
|
+
results.concat res
|
39
|
+
end
|
40
|
+
results
|
41
|
+
end
|
42
|
+
|
43
|
+
def total_queries
|
44
|
+
query.per_page(1).fetch unless query.total
|
45
|
+
(query.total / WorldBankFetcher::MAXIMUM_BUFFER_SIZE.to_f).ceil
|
46
|
+
end
|
47
|
+
|
48
|
+
def results=(results)
|
49
|
+
@results = results
|
50
|
+
end
|
51
|
+
|
52
|
+
def results
|
53
|
+
@results
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require "world_bank_fetcher/version"
|
2
|
+
require 'world_bank'
|
3
|
+
require 'world_bank_fetcher/job'
|
4
|
+
require 'world_bank_fetcher/query_scheduler'
|
5
|
+
require 'world_bank_fetcher/data_parser'
|
6
|
+
require 'world_bank_fetcher/country_parser'
|
7
|
+
|
8
|
+
module WorldBankFetcher
|
9
|
+
MAXIMUM_BUFFER_SIZE = 500
|
10
|
+
DEFAULT_INITIAL_BUFFER_SIZE = 50
|
11
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
Arab World
|
2
|
+
Caribbean small states
|
3
|
+
East Asia & Pacific (developing only)
|
4
|
+
East Asia & Pacific (all income levels)
|
5
|
+
Europe & Central Asia (developing only)
|
6
|
+
Europe & Central Asia (all income levels)
|
7
|
+
Euro area
|
8
|
+
European Union
|
9
|
+
High income
|
10
|
+
Heavily indebted poor countries (HIPC)
|
11
|
+
Not classified
|
12
|
+
Latin America & Caribbean (developing only)
|
13
|
+
Latin America & Caribbean (all income levels)
|
14
|
+
Least developed countries: UN classification
|
15
|
+
Low income
|
16
|
+
Lower middle income
|
17
|
+
Low & middle income
|
18
|
+
Middle East & North Africa (all income levels)
|
19
|
+
Middle income
|
20
|
+
Middle East & North Africa (developing only)
|
21
|
+
North America
|
22
|
+
High income: nonOECD
|
23
|
+
High income: OECD
|
24
|
+
OECD members
|
25
|
+
Other small states
|
26
|
+
Pacific island small states
|
27
|
+
South Asia
|
28
|
+
Sub-Saharan Africa (developing only)
|
29
|
+
Sub-Saharan Africa (all income levels)
|
30
|
+
Small states
|
31
|
+
Upper middle income
|
32
|
+
World
|
File without changes
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
describe CountryParser do
|
5
|
+
let(:france) { mock_country('France', "Paris", "Europe") }
|
6
|
+
let(:canada) { mock_country('Canada', "Ottawa", "North America") }
|
7
|
+
|
8
|
+
it "should parse a country object" do
|
9
|
+
expected = [{:name => "France", :capital => "Paris", :region => "Europe"}]
|
10
|
+
CountryParser.parse([france]).should eq(expected)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should parse multiple country objects" do
|
14
|
+
expected = [{:name => "France", :capital => "Paris", :region => "Europe"},
|
15
|
+
{:name => "Canada", :capital => "Ottawa", :region => "North America"} ]
|
16
|
+
CountryParser.parse([france, canada]).should eq(expected)
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
def mock_country(name, capital, region_name)
|
21
|
+
region_hash = Object.new
|
22
|
+
raw_object = Object.new
|
23
|
+
raw_object.stub(:value).and_return(region_name)
|
24
|
+
region_hash.stub(:raw).and_return(raw_object)
|
25
|
+
country = double('country')
|
26
|
+
country.stub(:name).and_return(name)
|
27
|
+
country.stub(:capital).and_return(capital)
|
28
|
+
country.stub(:region).and_return(region_hash)
|
29
|
+
country
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
describe DataParser do
|
5
|
+
let(:indicator_id) { 'DJ.JIE.JIDL' }
|
6
|
+
let(:datum1) { mock_data(:country => 'Mexico', :year => 1990, :id => indicator_id, :value => 234) }
|
7
|
+
let(:data) { [datum1] }
|
8
|
+
|
9
|
+
it "should parse data and return expected hash structure" do
|
10
|
+
actual = DataParser.parse data
|
11
|
+
actual.should eq({:id => indicator_id, 'Mexico' => [{:year => 1990, :value => 234}]})
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should return nil when provided data for multiple indicators" do
|
15
|
+
datum2 = mock_data(:country => 'Mexico', :year => 1990, :id => 'NJ.EKD.DKEL', :value => 234)
|
16
|
+
DataParser.parse([datum1, datum2]).should be_nil
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should handle more complext data" do
|
20
|
+
d1 = mock_data(:country => 'Mexico', :year => 1990, :id => indicator_id, :value => 234)
|
21
|
+
d2 = mock_data(:country => 'Mexico', :year => 1992, :id => indicator_id, :value => 456)
|
22
|
+
d3 = mock_data(:country => 'France', :year => 1990, :id => indicator_id, :value => 789)
|
23
|
+
expected = {:id => indicator_id, 'Mexico' => [{:year => 1990, :value => 234}, {:year => 1992, :value => 456}], 'France' => [{:year => 1990, :value => 789}]}
|
24
|
+
|
25
|
+
DataParser.parse([d1,d2,d3]).should eq(expected)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should mock data properly" do
|
29
|
+
f = data.first
|
30
|
+
f.value.should eq(234)
|
31
|
+
f.id.should eq('DJ.JIE.JIDL')
|
32
|
+
f.date.should eq(1990)
|
33
|
+
f.others['country']['value'].should eq('Mexico')
|
34
|
+
end
|
35
|
+
|
36
|
+
def mock_data(attr)
|
37
|
+
mockal = Object.new
|
38
|
+
mockal.stub(:value).and_return(attr[:value])
|
39
|
+
mockal.stub(:id).and_return(attr[:id])
|
40
|
+
mockal.stub(:date).and_return(attr[:year])
|
41
|
+
|
42
|
+
country_data = {'country' => {'value' => attr[:country]}}
|
43
|
+
mockal.stub(:others).and_return(country_data)
|
44
|
+
mockal
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/spec/helper.rb
ADDED
data/spec/job_spec.rb
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
describe Job do
|
5
|
+
let(:indicator_string) { 'SP.POP.TOTL' }
|
6
|
+
|
7
|
+
before do
|
8
|
+
DataParser.stub(:parse).and_return(:something)
|
9
|
+
end
|
10
|
+
|
11
|
+
context 'initialize' do
|
12
|
+
it "should accept hash w/ indicator to specify job type in intializer" do
|
13
|
+
WorldBank::Data.should_receive(:country).with('all').and_return(stubbed_query)
|
14
|
+
Job.new(:indicator => indicator_string)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should accept countries to indicate all countries in initializer hash" do
|
18
|
+
WorldBank::Country.should_receive(:all).and_return(stubbed_query)
|
19
|
+
Job.new(:countries => true)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
context 'fetch' do
|
24
|
+
it "should return nil if query shceduler returns nil" do
|
25
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
26
|
+
QueryScheduler.any_instance.should_receive(:execute!).and_return(nil)
|
27
|
+
job = Job.new(:indicator => indicator_string)
|
28
|
+
job.fetch.should be_nil
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should not return nil if query shceduler returns non nil value" do
|
32
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
33
|
+
QueryScheduler.any_instance.should_receive(:execute!).and_return(:something)
|
34
|
+
job = Job.new(:indicator => indicator_string)
|
35
|
+
job.fetch.should_not be_nil
|
36
|
+
end
|
37
|
+
|
38
|
+
it "returns a hash if there are no errors" do
|
39
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
40
|
+
QueryScheduler.any_instance.should_receive(:execute!).and_return(:something)
|
41
|
+
|
42
|
+
job = Job.new(:indicator => indicator_string)
|
43
|
+
job.fetch.should be_an_instance_of(Hash)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should receive same checksum if scheduler returns identical results" do
|
47
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
48
|
+
QueryScheduler.any_instance.stub(:execute!).and_return(:something)
|
49
|
+
|
50
|
+
job = Job.new(:indicator => indicator_string)
|
51
|
+
first = job.fetch[:checksum]
|
52
|
+
second = job.fetch[:checksum]
|
53
|
+
first.should eq(second)
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should return unique checksums given unique results" do
|
57
|
+
|
58
|
+
DataParser.should_receive(:parse).twice.and_return(:something, :something_else)
|
59
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
60
|
+
QueryScheduler.any_instance.stub(:execute!).and_return(:some_stuff)
|
61
|
+
|
62
|
+
job = Job.new(:indicator => indicator_string)
|
63
|
+
second_job = Job.new :indicator => 'different string'
|
64
|
+
first = job.fetch[:checksum]
|
65
|
+
second = second_job.fetch[:checksum]
|
66
|
+
first.should_not eq(second)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
def stubbed_query
|
72
|
+
dummy_query = double("query")
|
73
|
+
dummy_query.stub(:indicator)
|
74
|
+
dummy_query
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'world_bank'
|
3
|
+
|
4
|
+
module WorldBankFetcher
|
5
|
+
describe QueryScheduler do
|
6
|
+
let(:query) { WorldBank::Data.country('all').indicator('SP.POP.TOTL') }
|
7
|
+
let(:scheduler) { QueryScheduler.new(query) }
|
8
|
+
let(:fake_results) { [:some, :stuff, :goes, :here] }
|
9
|
+
|
10
|
+
before do
|
11
|
+
query.stub(:total).and_return(23)
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should trigger query fetch" do
|
15
|
+
query.should_receive(:fetch).at_least(:once)
|
16
|
+
scheduler.execute!
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should return nil if query fetch returns nil" do
|
20
|
+
query.stub(:fetch).and_return(nil)
|
21
|
+
scheduler.execute!.should be_nil
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should query once and return contents if total is not greater than max for already filled query" do
|
25
|
+
query.should_receive(:fetch).once.and_return(fake_results)
|
26
|
+
query.stub(:total).and_return(MAXIMUM_BUFFER_SIZE - 1)
|
27
|
+
fake_results = scheduler.execute!
|
28
|
+
fake_results.should eq(fake_results)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should query twice and return contents if total is just barely greater than max for already filled query" do
|
32
|
+
query.should_receive(:fetch).twice.and_return(fake_results)
|
33
|
+
query.stub(:total).and_return(MAXIMUM_BUFFER_SIZE + 1)
|
34
|
+
fake_results = scheduler.execute!
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should call preliminary fetch on query if total is nil, then full data query" do
|
38
|
+
query.should_receive(:total).and_return(nil)
|
39
|
+
query.should_receive(:fetch).twice.and_return(fake_results)
|
40
|
+
query.should_receive(:total).and_return(DEFAULT_INITIAL_BUFFER_SIZE - 1)
|
41
|
+
scheduler.execute!
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should call preliminary fetch on query if total is nil, then two full data queries given larger data set" do
|
45
|
+
query.should_receive(:total).and_return(nil)
|
46
|
+
query.should_receive(:fetch).exactly(3).times.and_return(fake_results)
|
47
|
+
query.should_receive(:total).and_return(MAXIMUM_BUFFER_SIZE + 1)
|
48
|
+
scheduler.execute!
|
49
|
+
end
|
50
|
+
|
51
|
+
it "execute should return single array of data" do
|
52
|
+
query.should_receive(:fetch).twice.and_return(fake_results)
|
53
|
+
query.stub(:total).and_return(MAXIMUM_BUFFER_SIZE + 1)
|
54
|
+
|
55
|
+
results = scheduler.execute!
|
56
|
+
results.should eq([:some, :stuff, :goes, :here, :some, :stuff, :goes, :here])
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should check for param queries, and raise if buffer maxed out" do
|
60
|
+
param_query = WorldBank::Country.all
|
61
|
+
param_query.should_not_receive(:total)
|
62
|
+
param_query.stub(:fetch).and_return(fake_results)
|
63
|
+
scheduler = QueryScheduler.new(param_query)
|
64
|
+
scheduler.execute!
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should fetch for param query exactly once" do
|
68
|
+
param_query = WorldBank::Country.all
|
69
|
+
param_query.should_receive(:fetch).once.and_return(fake_results)
|
70
|
+
scheduler = QueryScheduler.new(param_query)
|
71
|
+
scheduler.execute!
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should return nil if scheduler returns sardine packed results for param query" do
|
75
|
+
param_query = WorldBank::Country.all
|
76
|
+
sardines = Array(1..MAXIMUM_BUFFER_SIZE)
|
77
|
+
param_query.should_receive(:fetch).once.and_return(sardines)
|
78
|
+
scheduler = QueryScheduler.new(param_query)
|
79
|
+
scheduler.execute!.should be_nil
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/world_bank_fetcher/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Alexander Auritt"]
|
6
|
+
gem.email = ["alexauritt@gmail.com"]
|
7
|
+
gem.description = %q{For caching WorldBank data in a local db}
|
8
|
+
gem.summary = %q{Fetches local WorldBank country and indicator data.}
|
9
|
+
gem.homepage = ""
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "world_bank_fetcher"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = WorldBankFetcher::VERSION
|
17
|
+
|
18
|
+
gem.add_development_dependency "rspec"
|
19
|
+
gem.add_dependency "rake"
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: world_bank_fetcher
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Alexander Auritt
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-08-28 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rspec
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
description: For caching WorldBank data in a local db
|
47
|
+
email:
|
48
|
+
- alexauritt@gmail.com
|
49
|
+
executables: []
|
50
|
+
extensions: []
|
51
|
+
extra_rdoc_files: []
|
52
|
+
files:
|
53
|
+
- .gitignore
|
54
|
+
- .rvmrc
|
55
|
+
- Gemfile
|
56
|
+
- LICENSE
|
57
|
+
- README.md
|
58
|
+
- Rakefile
|
59
|
+
- country_list.txt
|
60
|
+
- lib/world_bank_fetcher.rb
|
61
|
+
- lib/world_bank_fetcher/country_parser.rb
|
62
|
+
- lib/world_bank_fetcher/data_parser.rb
|
63
|
+
- lib/world_bank_fetcher/job.rb
|
64
|
+
- lib/world_bank_fetcher/query_scheduler.rb
|
65
|
+
- lib/world_bank_fetcher/version.rb
|
66
|
+
- non_country_list.txt
|
67
|
+
- sorted_countries.txt
|
68
|
+
- spec/country_parser_spec.rb
|
69
|
+
- spec/data_parser_spec.rb
|
70
|
+
- spec/helper.rb
|
71
|
+
- spec/job_spec.rb
|
72
|
+
- spec/query_scheduler_spec.rb
|
73
|
+
- world_bank_fetcher.gemspec
|
74
|
+
homepage: ''
|
75
|
+
licenses: []
|
76
|
+
post_install_message:
|
77
|
+
rdoc_options: []
|
78
|
+
require_paths:
|
79
|
+
- lib
|
80
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
|
+
none: false
|
88
|
+
requirements:
|
89
|
+
- - ! '>='
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: '0'
|
92
|
+
requirements: []
|
93
|
+
rubyforge_project:
|
94
|
+
rubygems_version: 1.8.24
|
95
|
+
signing_key:
|
96
|
+
specification_version: 3
|
97
|
+
summary: Fetches local WorldBank country and indicator data.
|
98
|
+
test_files:
|
99
|
+
- spec/country_parser_spec.rb
|
100
|
+
- spec/data_parser_spec.rb
|
101
|
+
- spec/helper.rb
|
102
|
+
- spec/job_spec.rb
|
103
|
+
- spec/query_scheduler_spec.rb
|