world_bank_fetcher 0.0.12 → 0.0.13

data/lib/world_bank_fetcher.rb

@@ -3,6 +3,7 @@ require 'world_bank'
 require 'world_bank_fetcher/job'
 require 'world_bank_fetcher/query_scheduler'
 require 'world_bank_fetcher/country_parser'
+require 'world_bank_fetcher/indicator_data_parser'
 
 module WorldBankFetcher
   MAXIMUM_BUFFER_SIZE = 10000

data/lib/world_bank_fetcher/indicator_data_parser.rb

@@ -0,0 +1,10 @@
+require 'world_bank'
+
+module WorldBankFetcher
+  class IndicatorDataParser
+
+    def self.filter(data_collection)
+      data = data_collection.keep_if {|datum| datum.value != nil }
+    end
+  end
+end

data/lib/world_bank_fetcher/job.rb

@@ -14,15 +14,20 @@ module WorldBankFetcher
     def fetch
       all_data = fetch_everything query
       if all_data
-        data = CountryParser.filter(all_data)
-        data
+        data = @job_type == :country ? CountryParser.filter(all_data) : IndicatorDataParser.filter(all_data)
+        @checksum = checksum data
+        {:results => data, :checksum => @checksum}
       else
         nil
       end
     end
 
     private
-        
+    
+    def checksum(data)
+      Digest::MD5.hexdigest Marshal.dump(data)
+    end
+    
     def build_query(options)
       options[:indicator] ? indicator_query(options[:indicator]) : countries_query
     end

data/lib/world_bank_fetcher/version.rb

@@ -1,3 +1,3 @@
 module WorldBankFetcher
-  VERSION = "0.0.12"
+  VERSION = "0.0.13"
 end

data/spec/indicator_data_parser_spec.rb

@@ -0,0 +1,25 @@
+require 'helper'
+
+module WorldBankFetcher
+  describe IndicatorDataParser do
+    let(:other_attributes) { { }}
+    let(:good_data_1) { WorldBank::Data.new(attrs_with 'value' => 534) }
+    let(:good_data_2) { WorldBank::Data.new(attrs_with 'value' => 32) }
+    let(:bad_data_1) { WorldBank::Data.new(attrs_with 'value' => nil) }
+    let(:bad_data_2) { WorldBank::Data.new(attrs_with 'value' => nil) }
+
+    
+    it "should filter data with nil values" do
+      # CountryParser.filter([france, arab_world, canada]).should eq([france, canada])
+      good_data = [good_data_1, good_data_2]
+      IndicatorDataParser.filter([good_data_1, bad_data_2, bad_data_1, good_data_2]).should eq(good_data)
+    end
+    
+    private
+    
+    def attrs_with(attr)
+      {'indicator' => {'value' => '2', 'id' => 23}, 'date' => '1999'}.merge attr
+    end
+    
+  end
+end

data/spec/job_spec.rb

@@ -1,45 +1,98 @@
 require 'helper'
 
 module WorldBankFetcher
-  describe Job do    
+  describe Job do
+    before do
+      CountryParser.stub!(:filter) do |arg|
+        arg
+      end
+      
+      IndicatorDataParser.stub!(:filter) do |arg|
+        arg
+      end
+    end
+    
     let(:indicator_string) { 'SP.POP.TOTL' }
         
-    describe 'initialize' do
+    context 'initialize' do
       it "should accept hash w/ indicator to specify job type in intializer" do
-        country_parser_filters_nothing!
         WorldBank::Data.should_receive(:country).with('all').and_return(stubbed_query)
         Job.new(:indicator => indicator_string)
       end
       
       it "should accept countries to indicate all countries in initializer hash" do
-        country_parser_filters_nothing!
         WorldBank::Country.should_receive(:all).and_return(stubbed_query)
         Job.new(:countries => true)
       end
     end
 
-    describe 'fetch' do
+    context 'fetch' do
+      context 'filtering' do
+        it "should not use IndicatorDataParser to filter for country jobs" do
+          IndicatorDataParser.should_not_receive(:filter)
+          Job.new(:countries => true).fetch          
+        end
+        
+        it "should use IndicatorDataParser to filter for indicator jobs" do
+          WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
+          QueryScheduler.any_instance.should_receive(:execute!).and_return(:something)
+
+          IndicatorDataParser.should_receive(:filter)
+          Job.new(:indicator => indicator_string).fetch                  
+        end
+        
+        it "should not use CountryParser to filter for indicator jobs" do
+          WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
+          QueryScheduler.any_instance.should_receive(:execute!).and_return(:something)
+
+          CountryParser.should_not_receive(:filter)
+          Job.new(:indicator => indicator_string).fetch        
+        end
+      
+        it "should use CountryParser to filter for country jobs" do
+          CountryParser.should_receive(:filter)
+          Job.new(:countries => true).fetch
+        end
+      end
+      
       it "should return nil if query shceduler returns nil" do
-        country_parser_filters_nothing!
+        WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
         QueryScheduler.any_instance.should_receive(:execute!).and_return(nil)
         job = Job.new(:indicator => indicator_string)
         job.fetch.should be_nil
       end
 
       it "should not return nil if query shceduler returns non nil value" do
-        country_parser_filters_nothing!
+        WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
         QueryScheduler.any_instance.should_receive(:execute!).and_return(:something)
         job = Job.new(:indicator => indicator_string)
         job.fetch.should_not be_nil
       end
     
-      it "filters return value from Query Scheduler through CountryParser" do
+      it "returns a hash if there are no errors" do
+        WorldBank::DataQuery.any_instance.stub(:total).and_return(39)
         QueryScheduler.any_instance.should_receive(:execute!).and_return(:something)
-        CountryParser.should_receive(:filter).with(:something).and_return(:something_else)
+
         job = Job.new(:indicator => indicator_string)
-        job.fetch.should eq(:something_else)
+        job.fetch.should be_an_instance_of(Hash)
       end
-    end
+  
+      context 'checksum' do
+        it "should return unique checksums given unique results" do
+          job = Job.new(:indicator => indicator_string)
+          d1 = [:blah, :blog, :dj]
+          d2 = [:bjad, :awe, :asdf]
+          job.send(:checksum, d1).should_not eq(job.send(:checksum, d2))
+        end
+        
+        it "should receive same checksum if scheduler returns identical results" do
+          job = Job.new(:indicator => indicator_string)
+          d1 = [:blah, :blog, :dj]
+          job.send(:checksum, d1).should eq(job.send(:checksum, d1))          
+        end
+        
+      end
+    end    
     
     private
     def stubbed_query
@@ -47,11 +100,5 @@ module WorldBankFetcher
       dummy_query.stub(:indicator)
       dummy_query
     end
-    
-    def country_parser_filters_nothing!
-      CountryParser.stub!(:filter) do |arg|
-        arg
-      end
-    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: world_bank_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-31 00:00:00.000000000 Z
+date: 2012-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -61,11 +61,13 @@ files:
 - country_lists/sorted_countries.txt
 - lib/world_bank_fetcher.rb
 - lib/world_bank_fetcher/country_parser.rb
+- lib/world_bank_fetcher/indicator_data_parser.rb
 - lib/world_bank_fetcher/job.rb
 - lib/world_bank_fetcher/query_scheduler.rb
 - lib/world_bank_fetcher/version.rb
 - spec/country_parser_spec.rb
 - spec/helper.rb
+- spec/indicator_data_parser_spec.rb
 - spec/job_spec.rb
 - spec/query_scheduler_spec.rb
 - world_bank_fetcher.gemspec
@@ -83,7 +85,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4294862042739340307
+      hash: -4294314482202653082
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -92,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4294862042739340307
+      hash: -4294314482202653082
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -102,5 +104,6 @@ summary: Fetches local WorldBank country and indicator data.
 test_files:
 - spec/country_parser_spec.rb
 - spec/helper.rb
+- spec/indicator_data_parser_spec.rb
 - spec/job_spec.rb
 - spec/query_scheduler_spec.rb