wombat 2.4.0 → 2.5.0

data/lib/wombat/property/locators/factory.rb

@@ -11,32 +11,32 @@ require 'wombat/property/locators/headers'
 class Wombat::Property::Locators::UnknownTypeException < Exception; end;
 
 module Wombat
-	module Property
-		module Locators
-			module Factory
-				def self.locator_for(property)
-					klass = case(property.wombat_property_format)
-					when :text 
-						Text
-					when :list
-						List
-					when :html 
-						Html
-					when :iterator
-						Iterator
-					when :container
-						PropertyGroup
-					when :follow
-						Follow
-					when :headers
-						Headers
-					else 
-	      		raise Wombat::Property::Locators::UnknownTypeException.new("Unknown property format #{property.format}.")
-					end
+  module Property
+    module Locators
+      module Factory
+        def self.locator_for(property)
+          klass = case(property.wombat_property_format)
+          when :text
+            Text
+          when :list
+            List
+          when :html
+            Html
+          when :iterator
+            Iterator
+          when :container
+            PropertyGroup
+          when :follow
+            Follow
+          when :headers
+            Headers
+          else
+            raise Wombat::Property::Locators::UnknownTypeException.new("Unknown property format #{property.format}.")
+          end
 
-					klass.new(property)
-				end
-			end
-		end
-	end
-end
+          klass.new(property)
+        end
+      end
+    end
+  end
+end

data/lib/wombat/property/locators/follow.rb

@@ -1,46 +1,23 @@
 #coding: utf-8
 
 module Wombat
-	module Property
-	  module Locators
-	    class Follow < Base
-	    	def locate(context, page = nil)
-	    		super do
+  module Property
+    module Locators
+      class Follow < Base
+        def locate(context, page = nil)
+          super do
             locate_nodes(context).flat_map do |node|
-              retried = false
-              begin
-                # Certain erroneous pages contain http 
-                # links with relative href attribute, 
-                # while browsers actually use them as 
-                # absolute.
-                # So, let wombat try that approach when 
-                # loading relative link fails.
-                #
-                target_page = page.click node
-                context = target_page.parser
+              mechanize_page = context.mechanize_page
+              link = Mechanize::Page::Link.new(node, page, mechanize_page)
+              target_page = page.click link
+              context = target_page.parser
+              context.mechanize_page = mechanize_page
 
-                filter_properties(context, page)
-              rescue Mechanize::ResponseCodeError => e
-                # Either the page is unavailable, or
-                # the link is mistakenly relative
-                #
-                raise e if retried
-                
-                # Give it a try first time
-                href = node.attributes && node.attributes["href"]
-                if href.respond_to? :value
-                  href.value = '/' + href.value unless 
-                    href.value.start_with? '/'
-                  retried = true
-                  retry
-                else
-                  raise e
-                end
-              end
+              filter_properties(context, page)
             end
           end
-	    	end
-	    end
-	  end
-	end
+        end
+      end
+    end
+  end
 end

data/lib/wombat/property/locators/list.rb

@@ -1,17 +1,17 @@
 #coding: utf-8
 
 module Wombat
-	module Property
-	  module Locators
-	    class List < Base
-	    	def locate(context, page = nil)
-	    		super do
-	    			locate_nodes(context).map do |n| 
-	    				n.is_a?(String) ? n.strip : n.inner_text.strip
-	    			end
-	    		end
-	    	end
-	    end
-	  end
+  module Property
+    module Locators
+      class List < Base
+        def locate(context, page = nil)
+          super do
+            locate_nodes(context).map do |n|
+              n.is_a?(String) ? n.strip : n.inner_text.strip
+            end
+          end
+        end
+      end
+    end
   end
-end
+end

data/lib/wombat.rb

@@ -3,15 +3,15 @@
 require 'wombat/crawler'
 
 module Wombat
-	class << self
+  class << self
 
     attr_reader :proxy_args, :user_agent, :user_agent_alias
 
-		def crawl(&block)
-			klass = Class.new
-			klass.send(:include, Wombat::Crawler)
-			klass.new.crawl(&block)
-		end
+    def crawl(&block)
+      klass = Class.new
+      klass.send(:include, Wombat::Crawler)
+      klass.new.crawl(&block)
+    end
 
     def configure
       yield self
@@ -29,6 +29,6 @@ module Wombat
       @user_agent_alias = user_agent_alias
     end
 
-		alias_method :scrape, :crawl
-	end
+    alias_method :scrape, :crawl
+  end
 end

data/spec/crawler_spec.rb

@@ -13,12 +13,12 @@ describe Wombat::Crawler do
 
       @crawler.event { event_called = true }
 
-      event_called.should eq(true)
+      expect(event_called).to eq(true)
     end
 
     it 'should provide metadata to yielded block' do
       @crawler.event do
-        self.class.should == Wombat::DSL::PropertyGroup
+        self.class.should eq(Wombat::DSL::PropertyGroup)
       end
     end
 
@@ -36,11 +36,11 @@ describe Wombat::Crawler do
 
       @crawler.location { |v| v.latitude -50.2323 }
 
-      @crawler_instance.should_receive(:parse) do |arg|
-        arg["event"]["title"].selector.should == "Fulltronic Dezembro"
-        arg["event"]["time"].selector.to_s.should == time.to_s
-        arg["venue"]["name"].selector.should == "Scooba"
-        arg["location"]["latitude"].selector.should == -50.2323
+      expect(@crawler_instance).to receive(:parse) do |arg|
+        expect(arg["event"]["title"].selector).to eq("Fulltronic Dezembro")
+        expect(arg["event"]["time"].selector.to_s).to eq(time.to_s)
+        expect(arg["venue"]["name"].selector).to eq("Scooba")
+        expect(arg["location"]["latitude"].selector).to eq(-50.2323)
       end
 
       @crawler_instance.crawl
@@ -52,24 +52,28 @@ describe Wombat::Crawler do
       another_crawler_instance = another_crawler.new
 
       another_crawler.event { |e| e.title 'Ibiza' }
-      another_crawler_instance.should_receive(:parse) { |arg| arg["event"]["title"].selector.should == "Ibiza" }
+      expect(another_crawler_instance).to receive(:parse) { |arg|
+        expect(arg["event"]["title"].selector).to eq("Ibiza")
+      }
       another_crawler_instance.crawl
 
       @crawler.event { |e| e.title 'Fulltronic Dezembro' }
-      @crawler_instance.should_receive(:parse) { |arg| arg["event"]["title"].selector.should == "Fulltronic Dezembro" }
+      expect(@crawler_instance).to receive(:parse) { |arg|
+        expect(arg["event"]["title"].selector).to eq("Fulltronic Dezembro")
+      }
       @crawler_instance.crawl
     end
 
     it 'should be able to assign arbitrary plain text metadata' do
       @crawler.some_data("/event/list", :html, "geo") { |p| true }
 
-      @crawler_instance.should_receive(:parse) do |arg|
+      expect(@crawler_instance).to receive(:parse) do |arg|
         prop = arg['some_data']
-        prop.wombat_property_name.should == "some_data"
-        prop.selector.should == "/event/list"
-        prop.format.should == :html
-        prop.namespaces.should == "geo"
-        prop.callback.should_not be_nil
+        expect(prop.wombat_property_name).to eq("some_data")
+        expect(prop.selector).to eq("/event/list")
+        expect(prop.format).to eq(:html)
+        expect(prop.namespaces).to eq("geo")
+        expect(prop.callback).to_not eq(nil)
       end
 
       @crawler_instance.crawl
@@ -84,9 +88,9 @@ describe Wombat::Crawler do
         another "css=.information"
       end
 
-      @crawler_instance.should_receive(:parse) do |arg|
-        arg["structure"]["data"].selector.should == "xpath=/xyz"
-        arg["structure"]["another"].selector.should == "css=.information"
+      expect(@crawler_instance).to receive(:parse) do |arg|
+        expect(arg["structure"]["data"].selector).to eq("xpath=/xyz")
+        expect(arg["structure"]["another"].selector).to eq("css=.information")
       end
 
       @crawler_instance.crawl
@@ -97,8 +101,8 @@ describe Wombat::Crawler do
     end
 
     it 'should assign metadata format' do
-      @crawler_instance.should_receive(:parse) do |arg|
-        arg[:document_format].should == :xml
+      expect(@crawler_instance).to receive(:parse) do |arg|
+        expect(arg[:document_format]).to eq(:xml)
       end
       @crawler.document_format :xml
       @crawler_instance.crawl
@@ -108,9 +112,9 @@ describe Wombat::Crawler do
       @crawler.base_url "danielnc.com"
       @crawler.path "/itens"
 
-      @crawler_instance.should_receive(:parse) do |arg|
-        arg[:base_url].should == "danielnc.com"
-        arg[:path].should == "/itens/1"
+      expect(@crawler_instance).to receive(:parse) do |arg|
+        expect(arg[:base_url]).to eq("danielnc.com")
+        expect(arg[:path]).to eq("/itens/1")
       end
 
       @crawler_instance.crawl do
@@ -119,44 +123,59 @@ describe Wombat::Crawler do
 
       another_instance = @crawler.new
 
-      another_instance.should_receive(:parse) do |arg|
-        arg[:base_url].should == "danielnc.com"
-        arg[:path].should == "/itens"
+      expect(another_instance).to receive(:parse) do |arg|
+        expect(arg[:base_url]).to eq("danielnc.com")
+        expect(arg[:path]).to eq("/itens")
       end
 
       another_instance.crawl
     end
 
+    it 'should crawl with url and block' do
+      url = 'http://danielinc.com/itens'
+
+      expect(@crawler_instance).to receive(:parse).with(anything, url)
+      @crawler_instance.crawl(url) do
+      end
+
+      another_instance = @crawler.new
+      expect(another_instance).to receive(:parse).with(anything, url)
+
+      another_instance.crawl(url)
+    end
+
     it 'should remove created method missing' do
       @crawler.base_url "danielnc.com"
       @crawler.path "/itens"
 
-      @crawler_instance.should_receive(:parse) do |arg|
-        arg[:base_url].should == "danielnc.com"
-        arg[:path].should == "/itens/1"
+      expect(@crawler_instance).to receive(:parse) do |arg|
+        expect(arg[:base_url]).to eq("danielnc.com")
+        expect(arg[:path]).to eq("/itens/1")
       end
 
       @crawler_instance.crawl do
         path "/itens/1"
       end
 
-      lambda { @craler_intance.undefined_method }.should raise_error(NoMethodError)
+      expect(lambda {
+        @craler_intance.undefined_method
+      }).to raise_error(NoMethodError)
     end
 
     it 'should remove created instance variable' do
       @crawler.base_url "danielnc.com"
       @crawler.path "/itens"
 
-      @crawler_instance.should_receive(:parse) do |arg|
-        arg[:base_url].should == "danielnc.com"
-        arg[:path].should == "/itens/1"
+      expect(@crawler_instance).to receive(:parse) do |arg|
+        expect(arg[:base_url]).to eq("danielnc.com")
+        expect(arg[:path]).to eq("/itens/1")
       end
 
       @crawler_instance.crawl do
         path "/itens/1"
       end
 
-      @crawler_instance.instance_variables.index(:@metadata_dup).should be_nil
+      expect(@crawler_instance.instance_variables.index(:@metadata_dup)).to be_nil
     end
 
     context "response code" do
@@ -169,7 +188,7 @@ describe Wombat::Crawler do
           @crawler.search "css=.btn-search"
 
           @crawler_instance.crawl
-          @crawler_instance.response_code.should be(200)
+          expect(@crawler_instance.response_code).to be(200)
         end
 
       end
@@ -183,7 +202,7 @@ describe Wombat::Crawler do
           @crawler.document_format :xml
 
           @crawler_instance.crawl
-          @crawler_instance.response_code.should be(200)
+          expect(@crawler_instance.response_code).to be(200)
         end
       end
 
@@ -195,8 +214,9 @@ describe Wombat::Crawler do
 
           @crawler.search "css=.btn-search"
 
-          lambda { @crawler_instance.crawl }.should raise_error("404 => Net::HTTPNotFound for http://www.terra.com.br/portal/ -- unhandled response")
-          @crawler_instance.response_code.should be(404)
+          expect(lambda { @crawler_instance.crawl }).to raise_error(
+            "404 => Net::HTTPNotFound for http://www.terra.com.br/portal/ -- unhandled response")
+          expect(@crawler_instance.response_code).to be(404)
         end
       end
 
@@ -208,8 +228,10 @@ describe Wombat::Crawler do
 
           @crawler.search "css=.btn-search"
           @crawler.document_format :xml
-          lambda { @crawler_instance.crawl }.should raise_error(RestClient::ResourceNotFound)
-          @crawler_instance.response_code.should be(404)
+          expect(lambda {
+            @crawler_instance.crawl
+          }).to raise_error(RestClient::ResourceNotFound)
+          expect(@crawler_instance.response_code).to be(404)
         end
       end
     end
@@ -217,8 +239,8 @@ describe Wombat::Crawler do
 
   describe '#scrape' do
     it 'should alias to crawl' do
-      @crawler_instance.should_receive :parse
+      expect(@crawler_instance).to receive :parse
       @crawler_instance.scrape
     end
   end
-end
+end

data/spec/integration/crawler_inheritance_spec.rb

@@ -0,0 +1,63 @@
+# coding: utf-8
+require 'spec_helper'
+
+describe 'crawler base and one derived class' do
+  class A
+    include Wombat::Crawler
+    title 'xpath=//head/title'
+  end
+
+  class B < A
+    base_url "http://www.terra.com.br"
+    path "/portal"
+    search "css=.btn-search"
+  end
+
+  it 'extracts properties defined in the base class ' do
+    VCR.use_cassette('basic_crawler_page') do
+      b = B.new
+      data = b.crawl
+      expect(data).to have_key('title')
+      expect(data).to have_key('search')
+    end
+  end
+end
+
+describe 'two derived classes' do
+  class D
+    include Wombat::Crawler
+    title 'xpath=//head/title'
+  end
+
+  class E < D
+    base_url "http://www.terra.com.br"
+    path "/portal"
+    search "css=.btn-search"
+  end
+
+  class F < D
+    title 'xpath=//broken/badly'
+    base_url "https://www.github.com"
+    path "/"
+  end
+
+
+  it 'second derived class does not overwrite base class properties' do
+    VCR.use_cassette('basic_crawler_page') do
+      e = E.new
+      data = e.crawl
+      expect(data).to have_key('title')
+      expect(data).to have_key('search')
+      expect(data['title']).to eq('Terra - Notícias, vídeos, esportes, economia, diversão, música, moda, fotolog, blog, chat')
+    end
+  end
+
+  it 'first derived class does not overwrite base class properties' do
+    VCR.use_cassette('follow_links') do
+      f = F.new
+      data = f.crawl
+      expect(data).to have_key('title')
+      expect(data['title']).to be_nil
+    end
+  end
+end

data/spec/integration/integration_spec.rb

@@ -273,4 +273,34 @@ describe 'basic crawler setup' do
       }
     end
   end
+
+  it 'should make post requests if needed' do
+    VCR.use_cassette('make_post_request') do
+      data = { your_name: "Name" }
+      crawler = Class.new
+      crawler.send(:include, Wombat::Crawler)
+      crawler.base_url "http://hroch486.icpf.cas.cz"
+      crawler.path "/cgi-bin/echo.pl"
+      crawler.http_method :post
+      crawler.data data
+
+      crawler.my_name 'css=ul:last li:last'
+
+      crawler_instance = crawler.new
+      results = crawler_instance.crawl
+
+      results["my_name"].should eq("your_name = Name")
+    end
+  end
+
+  it 'should let the url be passed as an argument to crawl' do
+    VCR.use_cassette('basic_crawler_page') do
+      crawler = Class.new
+      crawler.send(:include, Wombat::Crawler)
+      crawler.send(:title, 'xpath=//head/title')
+      crawler_instance = crawler.new
+      results = crawler_instance.crawl('http://www.terra.com.br/portal')
+      results['title'].should eq('Terra - Notícias, vídeos, esportes, economia, diversão, música, moda, fotolog, blog, chat')
+    end
+  end
 end

data/spec/processing/parser_spec.rb

@@ -17,11 +17,46 @@ describe Wombat::Processing::Parser do
     fake_document.should_receive(:parser).and_return(fake_parser)
     fake_document.should_receive(:header).and_return(fake_header)
     fake_parser.should_receive(:headers=)
+    fake_parser.should_receive(:mechanize_page=)
     @parser.mechanize.should_receive(:get).with("http://www.google.com/search").and_return fake_document
 
     @parser.parse @metadata
   end
 
+  it 'should able to make post requests' do
+    data = { your_name: "Name" }
+    @metadata.base_url "http://hroch486.icpf.cas.cz"
+    @metadata.path "/cgi-bin/echo.pl"
+    @metadata.http_method :post
+    @metadata.data data
+
+    fake_document = double :document
+    fake_parser = double :parser
+    fake_header = double :header
+    fake_document.should_receive(:parser).and_return(fake_parser)
+    fake_document.should_receive(:header).and_return(fake_header)
+    fake_parser.should_receive(:headers=)
+    fake_parser.should_receive(:mechanize_page=)
+    @parser.mechanize.should_receive(:post).with("http://hroch486.icpf.cas.cz/cgi-bin/echo.pl", data).and_return fake_document
+
+    @parser.parse @metadata
+  end
+
+  it 'should accept the url as an argument to parse' do
+    @metadata.http_method :get
+
+    fake_document = double :document
+    fake_parser = double :parser
+    fake_header = double :header
+    fake_document.should_receive(:parser).and_return(fake_parser)
+    fake_document.should_receive(:header).and_return(fake_header)
+    fake_parser.should_receive(:headers=)
+    fake_parser.should_receive(:mechanize_page=)
+    @parser.mechanize.should_receive(:get).with("https://www.github.com/notifications").and_return fake_document
+
+    @parser.parse(@metadata, 'https://www.github.com/notifications')
+  end
+
   it 'should correctly parse xml documents' do
     fake_document = double :xml
     fake_parser = double :parser

data/spec/property/locators/factory_spec.rb

@@ -1,18 +1,18 @@
 require 'spec_helper'
 
 describe Wombat::Property::Locators::Factory do
-	it 'should instantiate correct locator according to property type' do
-		Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :text)).should be_a(Wombat::Property::Locators::Text)
-		Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :html)).should be_a(Wombat::Property::Locators::Html)
-		Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :list)).should be_a(Wombat::Property::Locators::List)
-		Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :follow)).should be_a(Wombat::Property::Locators::Follow)
-		Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :iterator)).should be_a(Wombat::Property::Locators::Iterator)
-		Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :container)).should be_a(Wombat::Property::Locators::PropertyGroup)
-	end
+  it 'should instantiate correct locator according to property type' do
+    Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :text)).should be_a(Wombat::Property::Locators::Text)
+    Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :html)).should be_a(Wombat::Property::Locators::Html)
+    Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :list)).should be_a(Wombat::Property::Locators::List)
+    Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :follow)).should be_a(Wombat::Property::Locators::Follow)
+    Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :iterator)).should be_a(Wombat::Property::Locators::Iterator)
+    Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :container)).should be_a(Wombat::Property::Locators::PropertyGroup)
+  end
 
-	it 'should raise correct exception if provided property is of unknown type' do
-		lambda {
-			Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :weird))
-		}.should raise_error(Wombat::Property::Locators::UnknownTypeException, "Unknown property format weird.")
-	end
-end
+  it 'should raise correct exception if provided property is of unknown type' do
+    lambda {
+      Wombat::Property::Locators::Factory.locator_for(Wombat::DSL::Property.new(nil, nil, :weird))
+    }.should raise_error(Wombat::Property::Locators::UnknownTypeException, "Unknown property format weird.")
+  end
+end

data/spec/property/locators/html_spec.rb

@@ -1,26 +1,26 @@
 require 'spec_helper'
 
 describe Wombat::Property::Locators::Html do
-	it 'should locate html property' do
-		fake_elem = double :element
-		context   = double :context
-		fake_elem.stub inner_html: "Something cool "
-		context.stub(:xpath).with("/abc", nil).and_return [fake_elem]
-		property = Wombat::DSL::Property.new('data1', 'xpath=/abc', :html)
+  it 'should locate html property' do
+    fake_elem = double :element
+    context   = double :context
+    fake_elem.stub inner_html: "Something cool "
+    context.stub(:xpath).with("/abc", nil).and_return [fake_elem]
+    property = Wombat::DSL::Property.new('data1', 'xpath=/abc', :html)
 
-		locator = Wombat::Property::Locators::Html.new(property)
+    locator = Wombat::Property::Locators::Html.new(property)
 
-		locator.locate(context).should == { "data1" => "Something cool" }
-	end
+    locator.locate(context).should == { "data1" => "Something cool" }
+  end
 
-	it 'should return null if the property cannot be found' do
-		fake_elem = double :element
-		context   = double :context
-		context.stub(:xpath).with("/abc", nil).and_return []
-		property = Wombat::DSL::Property.new('data1', 'xpath=/abc', :html)
+  it 'should return null if the property cannot be found' do
+    fake_elem = double :element
+    context   = double :context
+    context.stub(:xpath).with("/abc", nil).and_return []
+    property = Wombat::DSL::Property.new('data1', 'xpath=/abc', :html)
 
-		locator = Wombat::Property::Locators::Html.new(property)
+    locator = Wombat::Property::Locators::Html.new(property)
 
-		locator.locate(context).should == { "data1" => nil }
-	end
-end
+    locator.locate(context).should == { "data1" => nil }
+  end
+end

data/spec/property/locators/list_spec.rb

@@ -1,13 +1,13 @@
 require 'spec_helper'
 
 describe Wombat::Property::Locators::List do
-	it 'should locate a list of nodes' do
-		context   = double :context
-		context.stub(:css).with(".selector").and_return %w(1 2 3 4 5)
-		property = Wombat::DSL::Property.new('data1', 'css=.selector', :list)
+  it 'should locate a list of nodes' do
+    context   = double :context
+    context.stub(:css).with(".selector").and_return %w(1 2 3 4 5)
+    property = Wombat::DSL::Property.new('data1', 'css=.selector', :list)
 
-		locator = Wombat::Property::Locators::List.new(property)
+    locator = Wombat::Property::Locators::List.new(property)
 
-		locator.locate(context).should == { "data1" => %w(1 2 3 4 5) }
-	end
-end
+    locator.locate(context).should == { "data1" => %w(1 2 3 4 5) }
+  end
+end