extraloop 0.0.1

data/spec/fixtures/doc.json

@@ -0,0 +1 @@
+{"completed_in":0.241,"max_id":149792397398786048,"max_id_str":"149792397398786048","next_page":"?page=2&max_id=149792397398786048&q=lolcatz","page":1,"query":"lolcatz","refresh_url":"?since_id=149792397398786048&q=lolcatz","results":[{"created_at":"Thu, 22 Dec 2011 10:04:25 +0000","from_user":"ludovickohn","from_user_id":389760832,"from_user_id_str":"389760832","from_user_name":"Ludovic kohn","geo":null,"id":149792397398786048,"id_str":"149792397398786048","iso_language_code":"fr","metadata":{"result_type":"recent"},"profile_image_url":"http://a2.twimg.com/profile_images/1598373174/image_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1598373174/image_normal.jpg","source":"<a href="http://www.samsungmobile.com" rel="nofollow">Samsung Mobile</a>","text":"J'ai ramen\u00E9e mon #rat \u00E0 paris pour les vacances, mon #chat kiffe il a enfin la t\u00E9l\u00E9! #lolcatz http://t.co/DIvnew3o","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Thu, 22 Dec 2011 07:43:25 +0000","from_user":"proud_stang5","from_user_id":138298648,"from_user_id_str":"138298648","from_user_name":"Hunter Byrnes","geo":null,"id":149756915742806016,"id_str":"149756915742806016","iso_language_code":"de","metadata":{"result_type":"recent"},"profile_image_url":"http://a2.twimg.com/profile_images/1629863431/image_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1629863431/image_normal.jpg","source":"<a href="http://twitter.com/#!/download/iphone" rel="nofollow">Twitter for iPhone</a>","text":"@kenferrera88 #lolcatz girl","to_user":"kenferrera88","to_user_id":330123906,"to_user_id_str":"330123906","to_user_name":"McKenna Ferrrera","in_reply_to_status_id":149756483284893696,"in_reply_to_status_id_str":"149756483284893696"},{"created_at":"Thu, 22 Dec 2011 06:01:27 +0000","from_user":"CatherineSwa","from_user_id":28416785,"from_user_id_str":"28416785","from_user_name":"Catherine Swantner","geo":null,"id":149731252528889856,"id_str":"149731252528889856","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a3.twimg.com/profile_images/1462050284/5920354189_e564722964_b_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1462050284/5920354189_e564722964_b_normal.jpg","source":"<a href="http://twitter.com/">web</a>","text":"@nicole1brown lolcatz meow","to_user":"nicole1brown","to_user_id":328187799,"to_user_id_str":"328187799","to_user_name":"Nicole Brown","in_reply_to_status_id":149730450024308736,"in_reply_to_status_id_str":"149730450024308736"},{"created_at":"Thu, 22 Dec 2011 04:34:51 +0000","from_user":"DanielleKim3","from_user_id":442392597,"from_user_id_str":"442392597","from_user_name":"Danielle Kim","geo":null,"id":149709461068578816,"id_str":"149709461068578816","iso_language_code":"it","metadata":{"result_type":"recent"},"profile_image_url":"http://a0.twimg.com/profile_images/1705391173/319281_10150395530514216_619154215_8228458_1986790910_n_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1705391173/319281_10150395530514216_619154215_8228458_1986790910_n_normal.jpg","source":"<a href="http://twitter.com/#!/download/iphone" rel="nofollow">Twitter for iPhone</a>","text":"RT @LizzieMcCaffrey: LOLCATZ. #Iamscum http://t.co/EYU6xS4f","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Thu, 22 Dec 2011 03:48:42 +0000","from_user":"megg_xxo","from_user_id":360964239,"from_user_id_str":"360964239","from_user_name":"Megan Brown","geo":null,"id":149697846000619520,"id_str":"149697846000619520","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a2.twimg.com/profile_images/1610806404/image_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1610806404/image_normal.jpg","source":"<a href="http://twitter.com/#!/download/iphone" rel="nofollow">Twitter for iPhone</a>","text":"RT @sydney_berger96: rocking out to karaoke with @megg_xxo oh my god why are we so good #lolcatz","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Thu, 22 Dec 2011 03:36:46 +0000","from_user":"TeighanDavies","from_user_id":382347005,"from_user_id_str":"382347005","from_user_name":"Teighan Davies","geo":null,"id":149694844325990400,"id_str":"149694844325990400","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a2.twimg.com/profile_images/1613021110/Picture0298_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1613021110/Picture0298_normal.jpg","source":"<a href="http://twitter.com/#!/download/iphone" rel="nofollow">Twitter for iPhone</a>","text":"@stephmcduff Lolcatz no #butilove #DALLASGREEN","to_user":"stephmcduff","to_user_id":233011266,"to_user_id_str":"233011266","to_user_name":"Stephanie McDuff","in_reply_to_status_id":149694211095142401,"in_reply_to_status_id_str":"149694211095142401"},{"created_at":"Thu, 22 Dec 2011 03:15:32 +0000","from_user":"wolfdawg69","from_user_id":376549124,"from_user_id_str":"376549124","from_user_name":"Delilahhhhh","geo":null,"id":149689499847364608,"id_str":"149689499847364608","iso_language_code":"it","metadata":{"result_type":"recent"},"profile_image_url":"http://a3.twimg.com/profile_images/1551014969/bubbababby_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1551014969/bubbababby_normal.jpg","source":"<a href="http://twitter.com/devices" rel="nofollow">txt</a>","text":"LOLCATZ","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Thu, 22 Dec 2011 02:52:38 +0000","from_user":"PagingDrDre","from_user_id":335938237,"from_user_id_str":"335938237","from_user_name":"Andrea Ratzlaff","geo":null,"id":149683737448284162,"id_str":"149683737448284162","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a2.twimg.com/profile_images/1444343446/image_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1444343446/image_normal.jpg","source":"<a href="http://twitter.com/#!/download/iphone" rel="nofollow">Twitter for iPhone</a>","text":"LAWL lolcatz @t_chapstick14","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Thu, 22 Dec 2011 01:48:22 +0000","from_user":"BigBoiRono","from_user_id":341301684,"from_user_id_str":"341301684","from_user_name":"Donovan Huyck","geo":null,"id":149667564874768385,"id_str":"149667564874768385","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a3.twimg.com/profile_images/1457795278/synchro_meet_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1457795278/synchro_meet_normal.jpg","source":"<a href="http://twitter.com/devices" rel="nofollow">txt</a>","text":"Bai hai wut awks o SALT lolcatz #fuckyou","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Thu, 22 Dec 2011 00:11:38 +0000","from_user":"StephanieZOO","from_user_id":379177126,"from_user_id_str":"379177126","from_user_name":"Stephanie Zamora","geo":null,"id":149643220807192576,"id_str":"149643220807192576","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a1.twimg.com/profile_images/1557641924/035_06__s_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1557641924/035_06__s_normal.jpg","source":"<a href="http://dlvr.it" rel="nofollow">dlvr.it</a>","text":"LolCatz Navy Chooses Two Women To Share Traditional First Kiss","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Thu, 22 Dec 2011 00:11:02 +0000","from_user":"MissTawdry","from_user_id":284260713,"from_user_id_str":"284260713","from_user_name":"MissTawdry","geo":null,"id":149643068751093760,"id_str":"149643068751093760","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a0.twimg.com/profile_images/1431052668/bod_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1431052668/bod_normal.jpg","source":"<a href="http://twitterfeed.com" rel="nofollow">twitterfeed</a>","text":"#LolCatz Navy Chooses Two Women To Share Traditional First Kiss http://t.co/7VqFgQl0 @misslindadee","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Wed, 21 Dec 2011 23:51:17 +0000","from_user":"LIVInmylife6","from_user_id":266053733,"from_user_id_str":"266053733","from_user_name":"Olivia Lake$","geo":null,"id":149638097905258497,"id_str":"149638097905258497","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a1.twimg.com/profile_images/1703355659/IMG00351-20110523-2253_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1703355659/IMG00351-20110523-2253_normal.jpg","source":"<a href="http://blackberry.com/twitter" rel="nofollow">Twitter for BlackBerry\u00AE</a>","text":"@CelioAraujo10 lolcatz I guesss its ok...ps u should fight ross LOL","to_user":"CelioAraujo10","to_user_id":379857020,"to_user_id_str":"379857020","to_user_name":"Celio Araujo","in_reply_to_status_id":149637652952518657,"in_reply_to_status_id_str":"149637652952518657"},{"created_at":"Wed, 21 Dec 2011 19:05:48 +0000","from_user":"rl1971","from_user_id":390991730,"from_user_id_str":"390991730","from_user_name":"RL","geo":null,"id":149566253621723136,"id_str":"149566253621723136","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a3.twimg.com/profile_images/1588515141/twittericon_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1588515141/twittericon_normal.jpg","source":"<a href="http://www.tweetdeck.com" rel="nofollow">TweetDeck</a>","text":"@w1zz http://t.co/4aIMUK1E #lolcatz #liev Met credits naar @gnesmu :)","to_user":"w1zz","to_user_id":194056505,"to_user_id_str":"194056505","to_user_name":"WiZZarD","in_reply_to_status_id":149503807187984385,"in_reply_to_status_id_str":"149503807187984385"},{"created_at":"Wed, 21 Dec 2011 18:51:52 +0000","from_user":"xTalieee1171x","from_user_id":66912646,"from_user_id_str":"66912646","from_user_name":"TC McCullough","geo":null,"id":149562749293297666,"id_str":"149562749293297666","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http://a2.twimg.com/profile_images/1631481911/DSCN9149_normal.JPG","profile_image_url_https":"https://si0.twimg.com/profile_images/1631481911/DSCN9149_normal.JPG","source":"<a href="http://red-badger.com" rel="nofollow">Birdsong for Windows Phone</a>","text":"Cuuuuuute! RT @Skyrawr All of my animals chillin out! Lolcatz  http://t.co/inRSK1VM","to_user":null,"to_user_id":null,"to_user_id_str":null,"to_user_name":null},{"created_at":"Wed, 21 Dec 2011 18:12:26 +0000","from_user":"gentin123","from_user_id":279705653,"from_user_id_str":"279705653","from_user_name":"morgan booker","geo":null,"id":149552824877973504,"id_str":"149552824877973504","iso_language_code":"it","metadata":{"result_type":"recent"},"profile_image_url":"http://a1.twimg.com/profile_images/1664758777/39216_421872978366_729553366_5014440_2895898_n_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/1664758777/39216_421872978366_729553366_5014440_2895898_n_normal.jpg","source":"<a href="http://blackberry.com/twitter" rel="nofollow">Twitter for BlackBerry\u00AE</a>","text":"@mollyjayne_x everiiwon luffz LOLcatz http://t.co/UaGzxNtV","to_user":"mollyjayne_x","to_user_id":408582032,"to_user_id_str":"408582032","to_user_name":"molly","in_reply_to_status_id":149549826068066305,"in_reply_to_status_id_str":"149549826068066305"}],"results_per_page":15,"since_id":0,"since_id_str":"0"}

data/spec/helpers/scraper_helper.rb

@@ -0,0 +1,46 @@
+module Helpers
+  module Scrapers
+
+    # Public:
+    #
+    # Stubs a HTTP request/response pair
+    #
+    # request_args  - Hash of options to be passed to Typhoeus::Request
+    # response_args - Hash of options to be passed to Typhoeus::Response (and to Hydra#stub).
+    #
+    # Returns nothing.
+    #
+
+    def stub_http(request_args={}, response_args={})
+
+      response_args = {
+        :code => 200,
+        :headers => "Content-Type: text/html",
+        :body => "response stub"
+      }.merge(response_args)
+
+      request_args = {
+        :method => :get,
+        :url => anything,
+        :options => anything
+      }.merge(request_args)
+
+      @hydra ||= Typhoeus::Hydra.new
+      stub(Typhoeus::Hydra).new { @hydra }
+      response = Typhoeus::Response.new(response_args)
+
+      stub.proxy(Typhoeus::Request).new(request_args[:url], request_args[:options]) do |request|
+        #
+        # this allows to stub several requests by handing control to a block
+        #
+
+        if block_given?
+          yield(@hydra, request, response)
+        else
+          @hydra.stub(request_args[:method], request_args[:url]).and_return(response)
+        end
+        request
+      end
+    end
+  end
+end

data/spec/helpers/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'rr'
+require 'pry'
+
+base_path = File.expand_path(File.dirname(File.dirname(File.dirname(__FILE__))))
+
+load base_path + "/lib/extraloop.rb"
+
+require 'helpers/scraper_helper'
+
+RSpec.configure do |config|
+  config.mock_with :rr
+end

data/spec/iterative_scraper_spec.rb

@@ -0,0 +1,175 @@
+require 'helpers/spec_helper'
+include Helpers::Scrapers
+
+describe IterativeScraper do
+  before(:each) do
+    @fixture_doc ||= proc {
+      file = File.open("fixtures/doc.html", "r")
+      file_content = file.read
+      file.close
+      file_content
+    }.call
+  end
+
+  describe "#initialize" do
+    subject { IterativeScraper.new("http://whatever.net/search") }
+
+    it { subject.should be_a(ScraperBase) }
+    it { subject.should respond_to(:log) }
+  end
+
+  describe "#set_iteration" do
+    before do 
+      @scraper = IterativeScraper.new("http://whatever.net/search")
+    end
+
+    it "should allow passing a range and return itself" do
+      @scraper.set_iteration((0..10).step(5)).should be_an_instance_of(IterativeScraper)
+    end
+    it "should allow passing an array and return itself" do
+      @scraper.set_iteration([1, 2, 3, 4]).should be_an_instance_of(IterativeScraper)
+    end
+    it "should allow passing a string and a proc and return itself" do
+      @scraper.set_iteration("#pagination a", proc {}).should be_an_instance_of(IterativeScraper)
+    end
+  end
+
+  describe "#continue_with" do
+    before do 
+      @scraper = IterativeScraper.new("http://whatever.net/search")
+    end
+
+    subject { @scraper.continue_with( proc { |result| result['continue'] }) }
+    it { should be_an_instance_of(IterativeScraper) }
+  end
+
+
+  context "(single url pattern, iteration_set is range , async => false )" do
+    before(:each) do
+      @scraper = IterativeScraper.new("http://whatever.net/search")
+      mock(@scraper).run_super(:run).times(10) {}
+      @scraper.set_iteration(:p, (1..10))
+    end
+
+    describe "#run" do
+      it "super#run should be called 10 times" do
+        @scraper.run
+      end
+    end
+  end
+
+  context "(single url pattern, iteration_set is extractor, async => false )" do
+    before(:each) do
+
+      iteration_count = 0
+      @params_sent = []
+      iteration_proc = proc {[2, 3, 4]}
+
+      any_instance_of(ExtractionLoop) do |eloop|
+        stub(eloop).run {}
+      end
+
+      stub_http do |hydra, request, response|
+        hydra.stub(:get, /http:\/\/whatever\.net\/search/).and_return(response)
+        @params_sent << request.params[:p]
+      end
+
+      @scraper = IterativeScraper.
+        new("http://whatever.net/search-stuff").
+        set_iteration(:p, iteration_proc).
+        loop_on(".whatever").
+        set_hook(:data, proc { iteration_count += 1 }).
+        run()
+
+      @iteration_count = iteration_count
+    end
+
+    describe "#run" do
+      it "The :data hook should be called 4 times" do
+        @iteration_count.should eql(4)
+      end
+
+      it "should have sent p=1, p=2, p=3, p=4 as request parameters" do
+        @params_sent.should eql(["1", "2", "3", "4"])
+      end
+    end
+  end
+
+  context "(single url pattern, iteration_set is range, async => true )" do
+
+    before do
+      @params_sent = []
+      any_instance_of(ExtractionLoop) do |eloop|
+        stub(eloop).run {}
+      end
+
+      stub_http do |hydra, request, response|
+        hydra.stub(:get, request.url).and_return(response)
+        @params_sent << request.params[:p]
+      end
+
+      @scraper = IterativeScraper.
+        new("http://whatever.net/search", {:async => true}, {:params => {:format => "json"} } ).
+        set_iteration(:p, (0..20).step(5)).
+        loop_on(".whatever").
+        run()
+    end
+
+
+    describe "#run" do
+      it "params sent should be p=1, p=5, p=10, p=15, p=20" do
+        @params_sent.should eql([0, 5, 10, 15, 20].map &:to_s)
+      end
+    end
+  end
+
+  context "using #continue_with" do
+    
+    describe "#run" do
+      before do
+        continue_values = (5..10).to_a
+        @values_sent = []
+        shift_values = proc { |data| continue_values.shift }
+
+
+        stub_http({}, {headers: "Content-Type: application/json", :body => '{"hello":"test"}' }) do |hydra, request, response|
+          @values_sent << request.params[:continue]
+          hydra.stub(:get, request.url).and_return(response)
+        end
+
+        IterativeScraper.
+          new("http://twizzer.net/timeline").
+          loop_on(proc {}).
+          continue_with(:continue, shift_values).
+          run()
+
+        @continue_values = continue_values
+
+      end
+
+      #TODO: 
+      #
+      # When #continue_with is used, it would be better avoid sending
+      # an empty iteration_parameter
+      #
+      it "Should run 5 times", :failing => 'true' do
+        @continue_values.all? { |val| @values_sent.include? val.to_s }.should be_true
+      end
+
+    end
+  end
+
+  context "using #continue_with with async = true" do
+    describe "#run" do
+      before do
+        @scraper = IterativeScraper.
+          new("http://twizzer.net/timeline", :async => true)
+      end
+
+      it "should raise an exception" do
+        expect { @scraper.continue_with(:continue, proc {}) }.to raise_exception(IterativeScraper::Exceptions::NonGetAsyncRequestNotYetImplemented)
+      end
+    end
+  end
+
+end

data/spec/json_extractor_spec.rb

@@ -0,0 +1,146 @@
+require 'helpers/spec_helper'
+
+describe JsonExtractor do
+  before(:each) do
+    stub(scraper = Object.new).options
+    stub(scraper).results
+    @env = ExtractionEnvironment.new(scraper)
+    @json ||= lambda {
+      file = File.open('fixtures/doc.json', 'r')
+      content = file.read
+      file.close
+      content
+    }.call()
+  end
+
+  describe "#initialize" do
+    context("argument is a block") do
+      subject { JsonExtractor.new(:thing, @env, proc  {}) }
+
+      it { subject.field_name.should eql(:thing) }
+      it { should_not respond_to(:callback) }
+    end
+  end
+
+
+  describe "#extract_field" do
+
+    context "field_name and callback" do
+      before do
+        @extractor = JsonExtractor.new(:from_user, @env)
+        @node = @extractor.parse(@json)['results'].first
+      end
+
+      subject { @extractor.extract_field(@node) }
+      it { should eql("ludovickohn") }
+    end
+
+    context "field_name and callback" do
+      before do
+        @extractor = JsonExtractor.new(:from_user, @env, proc { |node| node['from_user_name'] } )
+        @node = @extractor.parse(@json)['results'].first
+      end
+
+      subject { @extractor.extract_field(@node) }
+      it { should eql("Ludovic kohn") }
+    end
+
+    context "field_name and attribute" do
+      before do
+        @extractor = JsonExtractor.new(:from_user, @env, :from_user_name )
+        @node = @extractor.parse(@json)['results'].first
+      end
+
+      subject { @extractor.extract_field(@node) }
+      it { should eql("Ludovic kohn") }
+    end
+
+    context "field name, attribute, and callback " do
+      before do
+        @extractor = JsonExtractor.new(:from_user, @env, :from_user_name, proc { |username| username.downcase.gsub("\s","-")  } )
+        @node = @extractor.parse(@json)['results'].first
+      end
+
+      subject { @extractor.extract_field(@node) }
+      it { should eql("ludovic-kohn") }
+    end
+
+    context("field name and array (see Utils::DeepFetchable)") do
+      before do 
+        @extractor = JsonExtractor.new(:from_user, @env, ['results', 0, 'from_user'])
+      end
+      subject {  @extractor.extract_field(@json) }
+      it { should eql("ludovickohn") }
+    end
+
+    context("field name, array, and callback") do
+      before do 
+        @extractor = JsonExtractor.new(:from_user, @env, ['results', 0, 'from_user'], proc { |username|  username.gsub("ckohn",'co') })
+      end
+      subject {  @extractor.extract_field(@json) }
+      it { should eql("ludovico") }
+    end
+
+  end
+
+  describe "#extract_list" do
+
+
+    context "using #get_in" do
+      before do
+        @extractor = JsonExtractor.new(nil, @env, ['results', 0..5])
+      end
+
+      subject { @extractor.extract_list(@json) }
+
+      it { subject.size.should eql(6) }
+    end
+
+    context "with json string input" do
+      before do
+        @extractor = JsonExtractor.new(nil, @env, proc { |data| data['results'] })
+      end
+
+      subject { @extractor.extract_list(@json) }
+      it { subject.size.should eql(15) }
+      it { should be_an_instance_of(Array) }
+    end
+
+    context "with pre-parsed input" do
+      before do
+        @extractor = JsonExtractor.new(nil, @env, proc { |data| data['results'] })
+      end
+
+      subject { @extractor.extract_list((Yajl::Parser.new).parse(@json)) }
+      it { subject.size.should eql(15) }
+      it { should be_an_instance_of(Array) }
+    end
+
+  end
+
+  context "non-string input" do
+    describe "#parse" do
+      before do 
+        @extractor = JsonExtractor.new(nil, @env, proc {})
+      end
+
+      it "Should raise an exception" do
+        expect { @extractor.parse(Nokogiri::HTML(@html)) }.to raise_exception(ExtractorBase::Exceptions::ExtractorParseError)
+      end
+    end
+  end
+
+  context "#json input" do
+    describe "#parse" do
+      before do
+        @extractor = JsonExtractor.new(nil, @env, proc {})
+      end
+
+      subject { @extractor.parse(@json) }
+
+      it { should respond_to(:get_in) }
+      it { should be_an_instance_of(Hash) }
+      it { should_not be_empty }
+    end
+  end
+end

data/spec/loggable_spec.rb

@@ -0,0 +1,25 @@
+require 'helpers/spec_helper'
+
+class LoggableClass
+  attr_reader :super_called
+  include Loggable
+
+  def initialize
+    @super_called=true
+  end
+end
+
+describe Loggable do
+  describe "#initialize" do
+
+    subject { LoggableClass.new }
+
+    it "should execute the class' #initialize method" do
+      subject.super_called.should eql(true)
+    end
+
+    it "should respond to the #log method" do
+      subject.respond_to?(:log).should be_true
+    end
+  end
+end

data/spec/scraper_base_spec.rb

@@ -0,0 +1,178 @@
+require 'helpers/spec_helper'
+include Helpers::Scrapers
+
+describe ScraperBase do
+  before do
+    @fixture_doc = File.open("fixtures/doc.html", 'r') do |file|
+      file.read
+    end
+  end
+
+  before(:each) do
+    @scraper = ScraperBase.new("http://localhost/fixture")
+  end
+
+
+  describe "#loop_on" do
+    subject { @scraper.loop_on("bla.bla") }
+    it { should be_an_instance_of(ScraperBase) }
+  end
+
+  describe "#extract" do
+    subject { @scraper.extract("fieldname", "bla.bla") }
+    it { should be_an_instance_of(ScraperBase) }
+  end
+
+  describe "#set_hook" do
+    subject { @scraper.set_hook(:after, proc {}) }
+    it { should be_an_instance_of(ScraperBase)  }
+  end
+
+  describe "#set_hook" do
+    it "should raise exception if no proc is provided" do
+      expect { @scraper.set_hook(:after, :method) }.to raise_exception(ScraperBase::Exceptions::HookArgumentError)
+    end
+  end
+
+  context "request params in both the url and the arguments hash" do
+    describe "#run" do
+      before do
+
+        @request_args = {}
+        url = "http://localhost/whatever?q=stuff&p=1&limit=100"
+        stub_http do |hydra, request, response|
+          @request_args = request.params
+          hydra.stub(:get, request.url).and_return(response)
+        end
+
+        any_instance_of(ExtractionLoop) do |extraloop|
+          stub(extraloop).run {}
+        end
+
+        @scraper = ScraperBase.new(url, {}, {
+          :params => { :limit => 250 }
+        }).loop_on(".stuff").run
+      end
+
+      it "should merge URL and request parameters" do
+        @request_args[:p].to_s.should eql("1")
+        @request_args[:q].to_s.should eql("stuff")
+        @request_args[:limit].to_s.should eql("250")
+      end 
+    end
+  end
+
+  context "single url, no options provided (async => false)" do
+    describe "#run" do
+      before do
+        @url = "http://localhost/fixture"
+        results = []
+
+        stub_http({}, :body => @fixture_doc) do |hydra, request, response|
+          hydra.stub(:get, request.url).and_return(response)
+        end
+
+        @scraper = ScraperBase.new(@url).
+          loop_on("ul li.file a").
+            extract(:url, :href).
+            extract(:filename).
+          set_hook(:data, proc { |records| records.each { |record| results << record }})
+
+        @results = results
+      end
+
+
+      it "Should handle response" do
+        @scraper.run
+        @results.should_not be_empty
+        @results.all? { |record| record.extracted_at && record.url && record.filename }.should be_true
+      end
+    end
+  end
+
+  context "multiple urls (async => false)" do
+    describe "#run" do
+      before do
+        @urls = [
+          "http://localhost/fixture1",
+          "http://localhost/fixture2",
+          "http://localhost/fixture3",
+        ]
+        results = []
+        @hydra_run_call_count = 0
+
+        stub_http do |hydra, request, response|
+          @urls.each { |url| hydra.stub(:get, url).and_return(response) }
+          stub.proxy(hydra).run { @hydra_run_call_count += 1  }
+        end
+
+        @scraper = ScraperBase.new(@urls, :log => false).
+          loop_on("ul li.file a").
+            extract(:url, :href).
+            extract(:filename).
+          set_hook(:data, proc { |records| records.each { |record| results << record } })
+
+        @results = results
+
+        @fake_loop = Object.new
+        stub(@fake_loop).run { }
+        stub(@fake_loop).environment { ExtractionEnvironment.new }
+        stub(@fake_loop).records { Array(1..3).map { |n| Object.new } }
+
+        mock(ExtractionLoop).new(is_a(DomExtractor), is_a(Array), is_a(String), is_a(Hash), is_a(ScraperBase)).times(3) { @fake_loop  }
+      end
+
+
+      it "Should handle response" do
+        @scraper.run
+        @results.size.should eql(9)
+        @hydra_run_call_count.should eql(@urls.size)
+      end
+    end
+  end
+
+
+  context "multiple urls (async => true)" do
+    describe "#run" do
+      before do
+        @urls = [
+          "http://localhost/fixture1",
+          "http://localhost/fixture2",
+          "http://localhost/fixture3",
+          "http://localhost/fixture4",
+          "http://localhost/fixture5",
+        ]
+        results = []
+        @hydra_run_call_count = 0
+
+        stub_http({}, :body => @fixture_doc) do |hydra, request, response|
+          @urls.each { |url| hydra.stub(:get, url).and_return(response) }
+          stub.proxy(hydra).run { @hydra_run_call_count+=1 }
+        end
+
+        @scraper = ScraperBase.new(@urls, :async => true).
+          loop_on("ul li.file a").
+            extract(:url, :href).
+            extract(:filename).
+          set_hook(:data, proc { |records| records.each { |record| results << record } })
+
+        @results = results
+
+        @fake_loop = Object.new
+        stub(@fake_loop).run { }
+        stub(@fake_loop).environment { ExtractionEnvironment.new }
+        stub(@fake_loop).records { Array(1..3).map { |n| Object.new } }
+
+        mock(ExtractionLoop).new(is_a(DomExtractor), is_a(Array), is_a(String), is_a(Hash), is_a(ScraperBase)).times(@urls.size) { @fake_loop  }
+      end
+
+
+      it "Should handle response" do
+        @scraper.run
+        @results.size.should eql(@urls.size * 3)
+        @hydra_run_call_count.should eql(1)
+      end
+    end
+  end
+
+end

data/spec/utils_spec.rb

@@ -0,0 +1,44 @@
+require 'helpers/spec_helper'
+
+describe Utils do
+  describe "AugmentedHash" do
+    describe "#get_in" do
+      context "extending a hash object" do
+        before do
+          @hash = {
+            :a => {
+              :b => {
+                :c => [
+                  :x, :y, :z
+                ]
+              }
+            }
+          }.extend(Utils::DeepFetchable)
+        end
+
+        subject { @hash.get_in [:a, :b, :c, 2] }
+
+        it { should eql(:z) }
+
+
+        context "trying to fetch a key that does not exist" do
+          subject { @hash.get_in [:a, :b, :wrong, :even_worst ]}
+
+          it { should eql(nil) }
+        end
+
+      end
+
+      context "extending an Array object" do
+        before do
+          @array = [1, 2, 3, 4, [5.1, 5.2, 5.3]].extend(Utils::DeepFetchable)
+        end
+
+
+        subject { @array.get_in [4, -1] }
+
+        it { should eql(5.3) }
+      end
+    end
+  end
+end