world_bank_fetcher 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +17 -0
- data/.rvmrc +1 -0
- data/Gemfile +6 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +21 -0
- data/country_list.txt +214 -0
- data/lib/world_bank_fetcher/country_parser.rb +17 -0
- data/lib/world_bank_fetcher/data_parser.rb +16 -0
- data/lib/world_bank_fetcher/job.rb +54 -0
- data/lib/world_bank_fetcher/query_scheduler.rb +56 -0
- data/lib/world_bank_fetcher/version.rb +3 -0
- data/lib/world_bank_fetcher.rb +11 -0
- data/non_country_list.txt +32 -0
- data/sorted_countries.txt +0 -0
- data/spec/country_parser_spec.rb +33 -0
- data/spec/data_parser_spec.rb +47 -0
- data/spec/helper.rb +2 -0
- data/spec/job_spec.rb +77 -0
- data/spec/query_scheduler_spec.rb +82 -0
- data/world_bank_fetcher.gemspec +20 -0
- metadata +103 -0
data/.gitignore
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm use 1.9.2@world_bank_fetcher
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Alexander Auritt
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# WorldBankFetcher
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'world_bank_fetcher'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install world_bank_fetcher
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require 'rspec/core/rake_task'
|
4
|
+
require './lib/world_bank_fetcher'
|
5
|
+
|
6
|
+
RSpec::Core::RakeTask.new(:spec)
|
7
|
+
# 'SP.POP.TOTL'
|
8
|
+
task :test => :spec
|
9
|
+
task :default => :spec
|
10
|
+
|
11
|
+
task :indicator, :indicator_code do |t, args|
|
12
|
+
# puts "code is #{args[:indicator_code]}"
|
13
|
+
puts WorldBankFetcher::Job.new(:indicator => args[:indicator_code]).fetch ? "QUERY SUCCESSFUL!" : "QUERY FAILURE!!!!!!!!!!"
|
14
|
+
end
|
15
|
+
|
16
|
+
task :countries do
|
17
|
+
results = WorldBankFetcher::Job.new(:countries => true).fetch
|
18
|
+
puts results ? "QUERY SUCCESSFUL!" : "QUERY FAILURE!!!!!!!!!!"
|
19
|
+
# puts "here are results"
|
20
|
+
puts "#{results}"
|
21
|
+
end
|
data/country_list.txt
ADDED
@@ -0,0 +1,214 @@
|
|
1
|
+
Aruba
|
2
|
+
Afghanistan
|
3
|
+
Angola
|
4
|
+
Albania
|
5
|
+
Andorra
|
6
|
+
United Arab Emirates
|
7
|
+
Argentina
|
8
|
+
Armenia
|
9
|
+
American Samoa
|
10
|
+
Antigua and Barbuda
|
11
|
+
Australia
|
12
|
+
Austria
|
13
|
+
Azerbaijan
|
14
|
+
Burundi
|
15
|
+
Belgium
|
16
|
+
Benin
|
17
|
+
Burkina Faso
|
18
|
+
Bangladesh
|
19
|
+
Bulgaria
|
20
|
+
Bahrain
|
21
|
+
Bahamas, The
|
22
|
+
Bosnia and Herzegovina
|
23
|
+
Belarus
|
24
|
+
Belize
|
25
|
+
Bermuda
|
26
|
+
Bolivia
|
27
|
+
Brazil
|
28
|
+
Barbados
|
29
|
+
Brunei Darussalam
|
30
|
+
Bhutan
|
31
|
+
Channel Islands
|
32
|
+
Hong Kong SAR, China
|
33
|
+
Botswana
|
34
|
+
Israel
|
35
|
+
Macao SAR, China
|
36
|
+
West Bank and Gaza
|
37
|
+
Central African Republic
|
38
|
+
Canada
|
39
|
+
Switzerland
|
40
|
+
Chile
|
41
|
+
China
|
42
|
+
Cote d'Ivoire
|
43
|
+
Cameroon
|
44
|
+
Congo, Dem. Rep.
|
45
|
+
Congo, Rep.
|
46
|
+
Colombia
|
47
|
+
Comoros
|
48
|
+
Cape Verde
|
49
|
+
Costa Rica
|
50
|
+
Cuba
|
51
|
+
Curacao
|
52
|
+
Cayman Islands
|
53
|
+
Cyprus
|
54
|
+
Czech Republic
|
55
|
+
Germany
|
56
|
+
Djibouti
|
57
|
+
Dominica
|
58
|
+
Denmark
|
59
|
+
Dominican Republic
|
60
|
+
Algeria
|
61
|
+
Ecuador
|
62
|
+
Egypt, Arab Rep.
|
63
|
+
Eritrea
|
64
|
+
Spain
|
65
|
+
Estonia
|
66
|
+
Ethiopia
|
67
|
+
Finland
|
68
|
+
Fiji
|
69
|
+
France
|
70
|
+
Faeroe Islands
|
71
|
+
Micronesia, Fed. Sts.
|
72
|
+
Gabon
|
73
|
+
United Kingdom
|
74
|
+
Georgia
|
75
|
+
Ghana
|
76
|
+
Guinea
|
77
|
+
Gambia, The
|
78
|
+
Guinea-Bissau
|
79
|
+
Equatorial Guinea
|
80
|
+
Greece
|
81
|
+
Grenada
|
82
|
+
Greenland
|
83
|
+
Guatemala
|
84
|
+
Guam
|
85
|
+
Guyana
|
86
|
+
Honduras
|
87
|
+
Croatia
|
88
|
+
Haiti
|
89
|
+
Hungary
|
90
|
+
Indonesia
|
91
|
+
Isle of Man
|
92
|
+
India
|
93
|
+
Ireland
|
94
|
+
Iran, Islamic Rep.
|
95
|
+
Iraq
|
96
|
+
Iceland
|
97
|
+
Italy
|
98
|
+
Jamaica
|
99
|
+
Jordan
|
100
|
+
Japan
|
101
|
+
Kazakhstan
|
102
|
+
Kenya
|
103
|
+
Kyrgyz Republic
|
104
|
+
Cambodia
|
105
|
+
Kiribati
|
106
|
+
St. Kitts and Nevis
|
107
|
+
Korea, Rep.
|
108
|
+
Kosovo
|
109
|
+
Kuwait
|
110
|
+
Lao PDR
|
111
|
+
Lebanon
|
112
|
+
Liberia
|
113
|
+
Libya
|
114
|
+
St. Lucia
|
115
|
+
Liechtenstein
|
116
|
+
Sri Lanka
|
117
|
+
Lesotho
|
118
|
+
Lithuania
|
119
|
+
Luxembourg
|
120
|
+
Latvia
|
121
|
+
St. Martin (French part)
|
122
|
+
Morocco
|
123
|
+
Monaco
|
124
|
+
Moldova
|
125
|
+
Madagascar
|
126
|
+
Maldives
|
127
|
+
Mexico
|
128
|
+
Marshall Islands
|
129
|
+
Macedonia, FYR
|
130
|
+
Mali
|
131
|
+
Malta
|
132
|
+
Myanmar
|
133
|
+
Montenegro
|
134
|
+
Mongolia
|
135
|
+
Northern Mariana Islands
|
136
|
+
Mozambique
|
137
|
+
Mauritania
|
138
|
+
Mauritius
|
139
|
+
Malawi
|
140
|
+
Malaysia
|
141
|
+
Namibia
|
142
|
+
New Caledonia
|
143
|
+
Niger
|
144
|
+
Nigeria
|
145
|
+
Nicaragua
|
146
|
+
Netherlands
|
147
|
+
Norway
|
148
|
+
Nepal
|
149
|
+
New Zealand
|
150
|
+
Oman
|
151
|
+
Pakistan
|
152
|
+
Panama
|
153
|
+
Peru
|
154
|
+
Philippines
|
155
|
+
Palau
|
156
|
+
Papua New Guinea
|
157
|
+
Poland
|
158
|
+
Puerto Rico
|
159
|
+
Korea, Dem. Rep.
|
160
|
+
Portugal
|
161
|
+
Paraguay
|
162
|
+
French Polynesia
|
163
|
+
Qatar
|
164
|
+
Romania
|
165
|
+
Russian Federation
|
166
|
+
Rwanda
|
167
|
+
Saudi Arabia
|
168
|
+
Sudan
|
169
|
+
Senegal
|
170
|
+
Singapore
|
171
|
+
Solomon Islands
|
172
|
+
Sierra Leone
|
173
|
+
El Salvador
|
174
|
+
San Marino
|
175
|
+
Somalia
|
176
|
+
Serbia
|
177
|
+
South Sudan
|
178
|
+
Sao Tome and Principe
|
179
|
+
Suriname
|
180
|
+
Slovak Republic
|
181
|
+
Slovenia
|
182
|
+
Sweden
|
183
|
+
Swaziland
|
184
|
+
Sint Maarten (Dutch part)
|
185
|
+
Seychelles
|
186
|
+
Syrian Arab Republic
|
187
|
+
Turks and Caicos Islands
|
188
|
+
Chad
|
189
|
+
Togo
|
190
|
+
Thailand
|
191
|
+
Tajikistan
|
192
|
+
Turkmenistan
|
193
|
+
Timor-Leste
|
194
|
+
Tonga
|
195
|
+
Trinidad and Tobago
|
196
|
+
Tunisia
|
197
|
+
Turkey
|
198
|
+
Tuvalu
|
199
|
+
Tanzania
|
200
|
+
Uganda
|
201
|
+
Ukraine
|
202
|
+
Uruguay
|
203
|
+
United States
|
204
|
+
Uzbekistan
|
205
|
+
St. Vincent and the Grenadines
|
206
|
+
Venezuela, RB
|
207
|
+
Virgin Islands (U.S.)
|
208
|
+
Vietnam
|
209
|
+
Vanuatu
|
210
|
+
Samoa
|
211
|
+
Yemen, Rep.
|
212
|
+
South Africa
|
213
|
+
Zambia
|
214
|
+
Zimbabwe
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'world_bank'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
class CountryParser
|
5
|
+
def self.parse(data_collection)
|
6
|
+
results = []
|
7
|
+
data_collection.each do |datum|
|
8
|
+
current_country = {}
|
9
|
+
current_country[:name] = datum.name
|
10
|
+
current_country[:capital] = datum.capital
|
11
|
+
current_country[:region] = datum.region.raw.value
|
12
|
+
results << current_country
|
13
|
+
end
|
14
|
+
results
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'world_bank'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
class DataParser
|
5
|
+
def self.parse(data_collection)
|
6
|
+
indicator_id = data_collection.first.id
|
7
|
+
results = {:id => indicator_id }
|
8
|
+
data_collection.each do |datum|
|
9
|
+
return nil unless datum.id === indicator_id
|
10
|
+
results[datum.others['country']['value']] ||= []
|
11
|
+
results[datum.others['country']['value']] << { :year => datum.date, :value => datum.value }
|
12
|
+
end
|
13
|
+
results
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'world_bank'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
class Job
|
5
|
+
attr_reader :results, :checksum
|
6
|
+
|
7
|
+
def initialize(options)
|
8
|
+
@results = nil
|
9
|
+
@checksum = nil
|
10
|
+
@job_type = options[:indicator] ? :indicator : :country
|
11
|
+
@query = build_query options
|
12
|
+
end
|
13
|
+
|
14
|
+
def fetch
|
15
|
+
data = fetch_all_data query
|
16
|
+
if data
|
17
|
+
if @job_type == :indicator
|
18
|
+
@results = WorldBankFetcher::DataParser.parse data
|
19
|
+
else
|
20
|
+
@results = WordBankFetcher::CountryParser.parse data
|
21
|
+
end
|
22
|
+
|
23
|
+
@checksum = Digest::MD5.hexdigest Marshal.dump(@results)
|
24
|
+
{:results => @results, :checksum => @checksum}
|
25
|
+
else
|
26
|
+
nil
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def build_query(options)
|
33
|
+
options[:indicator] ? indicator_query(options[:indicator]) : countries_query
|
34
|
+
end
|
35
|
+
|
36
|
+
def indicator_query(indicator_code)
|
37
|
+
WorldBank::Data.country('all').indicator(indicator_code)
|
38
|
+
end
|
39
|
+
|
40
|
+
def countries_query
|
41
|
+
WorldBank::Country.all
|
42
|
+
end
|
43
|
+
|
44
|
+
def query
|
45
|
+
@query
|
46
|
+
end
|
47
|
+
|
48
|
+
def fetch_all_data(query)
|
49
|
+
scheduler = WorldBankFetcher::QueryScheduler.new query
|
50
|
+
query.per_page(WorldBankFetcher::MAXIMUM_BUFFER_SIZE)
|
51
|
+
scheduler.execute!
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module WorldBankFetcher
|
2
|
+
class QueryScheduler
|
3
|
+
attr_reader :query
|
4
|
+
|
5
|
+
def initialize(query)
|
6
|
+
@query = query
|
7
|
+
@results = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def execute!
|
11
|
+
query.per_page(WorldBankFetcher::MAXIMUM_BUFFER_SIZE)
|
12
|
+
begin
|
13
|
+
if query.is_a? WorldBank::ParamQuery
|
14
|
+
fetch_param_query!
|
15
|
+
elsif query.is_a? WorldBank::DataQuery
|
16
|
+
fetch_data_query!
|
17
|
+
else
|
18
|
+
raise
|
19
|
+
end
|
20
|
+
rescue
|
21
|
+
nil
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def fetch_param_query!
|
28
|
+
results = query.fetch
|
29
|
+
rails if results.size === MAXIMUM_BUFFER_SIZE
|
30
|
+
results
|
31
|
+
end
|
32
|
+
|
33
|
+
def fetch_data_query!
|
34
|
+
results = []
|
35
|
+
total_queries.times do |i|
|
36
|
+
res = query.page(i+1).fetch
|
37
|
+
return nil if res.nil?
|
38
|
+
results.concat res
|
39
|
+
end
|
40
|
+
results
|
41
|
+
end
|
42
|
+
|
43
|
+
def total_queries
|
44
|
+
query.per_page(1).fetch unless query.total
|
45
|
+
(query.total / WorldBankFetcher::MAXIMUM_BUFFER_SIZE.to_f).ceil
|
46
|
+
end
|
47
|
+
|
48
|
+
def results=(results)
|
49
|
+
@results = results
|
50
|
+
end
|
51
|
+
|
52
|
+
def results
|
53
|
+
@results
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require "world_bank_fetcher/version"
|
2
|
+
require 'world_bank'
|
3
|
+
require 'world_bank_fetcher/job'
|
4
|
+
require 'world_bank_fetcher/query_scheduler'
|
5
|
+
require 'world_bank_fetcher/data_parser'
|
6
|
+
require 'world_bank_fetcher/country_parser'
|
7
|
+
|
8
|
+
module WorldBankFetcher
|
9
|
+
MAXIMUM_BUFFER_SIZE = 500
|
10
|
+
DEFAULT_INITIAL_BUFFER_SIZE = 50
|
11
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
Arab World
|
2
|
+
Caribbean small states
|
3
|
+
East Asia & Pacific (developing only)
|
4
|
+
East Asia & Pacific (all income levels)
|
5
|
+
Europe & Central Asia (developing only)
|
6
|
+
Europe & Central Asia (all income levels)
|
7
|
+
Euro area
|
8
|
+
European Union
|
9
|
+
High income
|
10
|
+
Heavily indebted poor countries (HIPC)
|
11
|
+
Not classified
|
12
|
+
Latin America & Caribbean (developing only)
|
13
|
+
Latin America & Caribbean (all income levels)
|
14
|
+
Least developed countries: UN classification
|
15
|
+
Low income
|
16
|
+
Lower middle income
|
17
|
+
Low & middle income
|
18
|
+
Middle East & North Africa (all income levels)
|
19
|
+
Middle income
|
20
|
+
Middle East & North Africa (developing only)
|
21
|
+
North America
|
22
|
+
High income: nonOECD
|
23
|
+
High income: OECD
|
24
|
+
OECD members
|
25
|
+
Other small states
|
26
|
+
Pacific island small states
|
27
|
+
South Asia
|
28
|
+
Sub-Saharan Africa (developing only)
|
29
|
+
Sub-Saharan Africa (all income levels)
|
30
|
+
Small states
|
31
|
+
Upper middle income
|
32
|
+
World
|
File without changes
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
describe CountryParser do
|
5
|
+
let(:france) { mock_country('France', "Paris", "Europe") }
|
6
|
+
let(:canada) { mock_country('Canada', "Ottawa", "North America") }
|
7
|
+
|
8
|
+
it "should parse a country object" do
|
9
|
+
expected = [{:name => "France", :capital => "Paris", :region => "Europe"}]
|
10
|
+
CountryParser.parse([france]).should eq(expected)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should parse multiple country objects" do
|
14
|
+
expected = [{:name => "France", :capital => "Paris", :region => "Europe"},
|
15
|
+
{:name => "Canada", :capital => "Ottawa", :region => "North America"} ]
|
16
|
+
CountryParser.parse([france, canada]).should eq(expected)
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
def mock_country(name, capital, region_name)
|
21
|
+
region_hash = Object.new
|
22
|
+
raw_object = Object.new
|
23
|
+
raw_object.stub(:value).and_return(region_name)
|
24
|
+
region_hash.stub(:raw).and_return(raw_object)
|
25
|
+
country = double('country')
|
26
|
+
country.stub(:name).and_return(name)
|
27
|
+
country.stub(:capital).and_return(capital)
|
28
|
+
country.stub(:region).and_return(region_hash)
|
29
|
+
country
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
describe DataParser do
|
5
|
+
let(:indicator_id) { 'DJ.JIE.JIDL' }
|
6
|
+
let(:datum1) { mock_data(:country => 'Mexico', :year => 1990, :id => indicator_id, :value => 234) }
|
7
|
+
let(:data) { [datum1] }
|
8
|
+
|
9
|
+
it "should parse data and return expected hash structure" do
|
10
|
+
actual = DataParser.parse data
|
11
|
+
actual.should eq({:id => indicator_id, 'Mexico' => [{:year => 1990, :value => 234}]})
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should return nil when provided data for multiple indicators" do
|
15
|
+
datum2 = mock_data(:country => 'Mexico', :year => 1990, :id => 'NJ.EKD.DKEL', :value => 234)
|
16
|
+
DataParser.parse([datum1, datum2]).should be_nil
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should handle more complext data" do
|
20
|
+
d1 = mock_data(:country => 'Mexico', :year => 1990, :id => indicator_id, :value => 234)
|
21
|
+
d2 = mock_data(:country => 'Mexico', :year => 1992, :id => indicator_id, :value => 456)
|
22
|
+
d3 = mock_data(:country => 'France', :year => 1990, :id => indicator_id, :value => 789)
|
23
|
+
expected = {:id => indicator_id, 'Mexico' => [{:year => 1990, :value => 234}, {:year => 1992, :value => 456}], 'France' => [{:year => 1990, :value => 789}]}
|
24
|
+
|
25
|
+
DataParser.parse([d1,d2,d3]).should eq(expected)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should mock data properly" do
|
29
|
+
f = data.first
|
30
|
+
f.value.should eq(234)
|
31
|
+
f.id.should eq('DJ.JIE.JIDL')
|
32
|
+
f.date.should eq(1990)
|
33
|
+
f.others['country']['value'].should eq('Mexico')
|
34
|
+
end
|
35
|
+
|
36
|
+
def mock_data(attr)
|
37
|
+
mockal = Object.new
|
38
|
+
mockal.stub(:value).and_return(attr[:value])
|
39
|
+
mockal.stub(:id).and_return(attr[:id])
|
40
|
+
mockal.stub(:date).and_return(attr[:year])
|
41
|
+
|
42
|
+
country_data = {'country' => {'value' => attr[:country]}}
|
43
|
+
mockal.stub(:others).and_return(country_data)
|
44
|
+
mockal
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/spec/helper.rb
ADDED
data/spec/job_spec.rb
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module WorldBankFetcher
|
4
|
+
describe Job do
|
5
|
+
let(:indicator_string) { 'SP.POP.TOTL' }
|
6
|
+
|
7
|
+
before do
|
8
|
+
DataParser.stub(:parse).and_return(:something)
|
9
|
+
end
|
10
|
+
|
11
|
+
context 'initialize' do
|
12
|
+
it "should accept hash w/ indicator to specify job type in intializer" do
|
13
|
+
WorldBank::Data.should_receive(:country).with('all').and_return(stubbed_query)
|
14
|
+
Job.new(:indicator => indicator_string)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should accept countries to indicate all countries in initializer hash" do
|
18
|
+
WorldBank::Country.should_receive(:all).and_return(stubbed_query)
|
19
|
+
Job.new(:countries => true)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
context 'fetch' do
|
24
|
+
it "should return nil if query shceduler returns nil" do
|
25
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
26
|
+
QueryScheduler.any_instance.should_receive(:execute!).and_return(nil)
|
27
|
+
job = Job.new(:indicator => indicator_string)
|
28
|
+
job.fetch.should be_nil
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should not return nil if query shceduler returns non nil value" do
|
32
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
33
|
+
QueryScheduler.any_instance.should_receive(:execute!).and_return(:something)
|
34
|
+
job = Job.new(:indicator => indicator_string)
|
35
|
+
job.fetch.should_not be_nil
|
36
|
+
end
|
37
|
+
|
38
|
+
it "returns a hash if there are no errors" do
|
39
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
40
|
+
QueryScheduler.any_instance.should_receive(:execute!).and_return(:something)
|
41
|
+
|
42
|
+
job = Job.new(:indicator => indicator_string)
|
43
|
+
job.fetch.should be_an_instance_of(Hash)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should receive same checksum if scheduler returns identical results" do
|
47
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
48
|
+
QueryScheduler.any_instance.stub(:execute!).and_return(:something)
|
49
|
+
|
50
|
+
job = Job.new(:indicator => indicator_string)
|
51
|
+
first = job.fetch[:checksum]
|
52
|
+
second = job.fetch[:checksum]
|
53
|
+
first.should eq(second)
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should return unique checksums given unique results" do
|
57
|
+
|
58
|
+
DataParser.should_receive(:parse).twice.and_return(:something, :something_else)
|
59
|
+
WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
|
60
|
+
QueryScheduler.any_instance.stub(:execute!).and_return(:some_stuff)
|
61
|
+
|
62
|
+
job = Job.new(:indicator => indicator_string)
|
63
|
+
second_job = Job.new :indicator => 'different string'
|
64
|
+
first = job.fetch[:checksum]
|
65
|
+
second = second_job.fetch[:checksum]
|
66
|
+
first.should_not eq(second)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
def stubbed_query
|
72
|
+
dummy_query = double("query")
|
73
|
+
dummy_query.stub(:indicator)
|
74
|
+
dummy_query
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'world_bank'
|
3
|
+
|
4
|
+
module WorldBankFetcher
|
5
|
+
describe QueryScheduler do
|
6
|
+
let(:query) { WorldBank::Data.country('all').indicator('SP.POP.TOTL') }
|
7
|
+
let(:scheduler) { QueryScheduler.new(query) }
|
8
|
+
let(:fake_results) { [:some, :stuff, :goes, :here] }
|
9
|
+
|
10
|
+
before do
|
11
|
+
query.stub(:total).and_return(23)
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should trigger query fetch" do
|
15
|
+
query.should_receive(:fetch).at_least(:once)
|
16
|
+
scheduler.execute!
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should return nil if query fetch returns nil" do
|
20
|
+
query.stub(:fetch).and_return(nil)
|
21
|
+
scheduler.execute!.should be_nil
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should query once and return contents if total is not greater than max for already filled query" do
|
25
|
+
query.should_receive(:fetch).once.and_return(fake_results)
|
26
|
+
query.stub(:total).and_return(MAXIMUM_BUFFER_SIZE - 1)
|
27
|
+
fake_results = scheduler.execute!
|
28
|
+
fake_results.should eq(fake_results)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should query twice and return contents if total is just barely greater than max for already filled query" do
|
32
|
+
query.should_receive(:fetch).twice.and_return(fake_results)
|
33
|
+
query.stub(:total).and_return(MAXIMUM_BUFFER_SIZE + 1)
|
34
|
+
fake_results = scheduler.execute!
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should call preliminary fetch on query if total is nil, then full data query" do
|
38
|
+
query.should_receive(:total).and_return(nil)
|
39
|
+
query.should_receive(:fetch).twice.and_return(fake_results)
|
40
|
+
query.should_receive(:total).and_return(DEFAULT_INITIAL_BUFFER_SIZE - 1)
|
41
|
+
scheduler.execute!
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should call preliminary fetch on query if total is nil, then two full data queries given larger data set" do
|
45
|
+
query.should_receive(:total).and_return(nil)
|
46
|
+
query.should_receive(:fetch).exactly(3).times.and_return(fake_results)
|
47
|
+
query.should_receive(:total).and_return(MAXIMUM_BUFFER_SIZE + 1)
|
48
|
+
scheduler.execute!
|
49
|
+
end
|
50
|
+
|
51
|
+
it "execute should return single array of data" do
|
52
|
+
query.should_receive(:fetch).twice.and_return(fake_results)
|
53
|
+
query.stub(:total).and_return(MAXIMUM_BUFFER_SIZE + 1)
|
54
|
+
|
55
|
+
results = scheduler.execute!
|
56
|
+
results.should eq([:some, :stuff, :goes, :here, :some, :stuff, :goes, :here])
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should check for param queries, and raise if buffer maxed out" do
|
60
|
+
param_query = WorldBank::Country.all
|
61
|
+
param_query.should_not_receive(:total)
|
62
|
+
param_query.stub(:fetch).and_return(fake_results)
|
63
|
+
scheduler = QueryScheduler.new(param_query)
|
64
|
+
scheduler.execute!
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should fetch for param query exactly once" do
|
68
|
+
param_query = WorldBank::Country.all
|
69
|
+
param_query.should_receive(:fetch).once.and_return(fake_results)
|
70
|
+
scheduler = QueryScheduler.new(param_query)
|
71
|
+
scheduler.execute!
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should return nil if scheduler returns sardine packed results for param query" do
|
75
|
+
param_query = WorldBank::Country.all
|
76
|
+
sardines = Array(1..MAXIMUM_BUFFER_SIZE)
|
77
|
+
param_query.should_receive(:fetch).once.and_return(sardines)
|
78
|
+
scheduler = QueryScheduler.new(param_query)
|
79
|
+
scheduler.execute!.should be_nil
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/world_bank_fetcher/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Alexander Auritt"]
|
6
|
+
gem.email = ["alexauritt@gmail.com"]
|
7
|
+
gem.description = %q{For caching WorldBank data in a local db}
|
8
|
+
gem.summary = %q{Fetches local WorldBank country and indicator data.}
|
9
|
+
gem.homepage = ""
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "world_bank_fetcher"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = WorldBankFetcher::VERSION
|
17
|
+
|
18
|
+
gem.add_development_dependency "rspec"
|
19
|
+
gem.add_dependency "rake"
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: world_bank_fetcher
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Alexander Auritt
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-08-28 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rspec
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
description: For caching WorldBank data in a local db
|
47
|
+
email:
|
48
|
+
- alexauritt@gmail.com
|
49
|
+
executables: []
|
50
|
+
extensions: []
|
51
|
+
extra_rdoc_files: []
|
52
|
+
files:
|
53
|
+
- .gitignore
|
54
|
+
- .rvmrc
|
55
|
+
- Gemfile
|
56
|
+
- LICENSE
|
57
|
+
- README.md
|
58
|
+
- Rakefile
|
59
|
+
- country_list.txt
|
60
|
+
- lib/world_bank_fetcher.rb
|
61
|
+
- lib/world_bank_fetcher/country_parser.rb
|
62
|
+
- lib/world_bank_fetcher/data_parser.rb
|
63
|
+
- lib/world_bank_fetcher/job.rb
|
64
|
+
- lib/world_bank_fetcher/query_scheduler.rb
|
65
|
+
- lib/world_bank_fetcher/version.rb
|
66
|
+
- non_country_list.txt
|
67
|
+
- sorted_countries.txt
|
68
|
+
- spec/country_parser_spec.rb
|
69
|
+
- spec/data_parser_spec.rb
|
70
|
+
- spec/helper.rb
|
71
|
+
- spec/job_spec.rb
|
72
|
+
- spec/query_scheduler_spec.rb
|
73
|
+
- world_bank_fetcher.gemspec
|
74
|
+
homepage: ''
|
75
|
+
licenses: []
|
76
|
+
post_install_message:
|
77
|
+
rdoc_options: []
|
78
|
+
require_paths:
|
79
|
+
- lib
|
80
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
|
+
none: false
|
88
|
+
requirements:
|
89
|
+
- - ! '>='
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: '0'
|
92
|
+
requirements: []
|
93
|
+
rubyforge_project:
|
94
|
+
rubygems_version: 1.8.24
|
95
|
+
signing_key:
|
96
|
+
specification_version: 3
|
97
|
+
summary: Fetches local WorldBank country and indicator data.
|
98
|
+
test_files:
|
99
|
+
- spec/country_parser_spec.rb
|
100
|
+
- spec/data_parser_spec.rb
|
101
|
+
- spec/helper.rb
|
102
|
+
- spec/job_spec.rb
|
103
|
+
- spec/query_scheduler_spec.rb
|