wombat 2.1.3 → 2.2.0

data/VERSION

@@ -1 +1 @@
-2.1.3
+2.2.0

data/lib/wombat/dsl/metadata.rb

@@ -23,6 +23,10 @@ module Wombat
       def document_format(format)
         self[:document_format] = format
       end
+
+      def page(page)
+        self[:page] = page
+      end
     end
   end
-end
+end

data/lib/wombat/processing/parser.rb

@@ -34,12 +34,14 @@ module Wombat
         page = nil
         parser = nil
         begin
+          @page = metadata[:page]
+
           if metadata[:document_format] == :html
-            @page = @mechanize.get(url)
+            @page = @mechanize.get(url) unless @page
             parser = @page.parser
             parser.headers = @page.header
           else
-            @page = RestClient.get(url)
+            @page = RestClient.get(url) unless @page
             parser = Nokogiri::XML @page
             parser.headers = @page.headers
           end
@@ -56,4 +58,4 @@ module Wombat
       end
     end
   end
-end
+end

data/spec/integration/integration_spec.rb

@@ -32,6 +32,37 @@ describe 'basic crawler setup' do
     end
   end
 
+  it 'should crawl a Mechanize::Page' do
+    VCR.use_cassette('basic_crawler_page') do
+      crawler = Class.new
+      crawler.send(:include, Wombat::Crawler)
+
+      m = Mechanize.new
+      mp = m.get "http://www.terra.com.br/portal"
+      crawler.page mp
+
+      crawler.search "css=.btn-search"
+      crawler.social do
+        twitter "css=.ctn-bar li.last"
+      end
+      crawler.links "css=.ctn-links", :iterator do
+        menu "css=a"
+      end
+      crawler.subheader "css=h2.ttl-dynamic" do |h|
+        h.gsub("London", "Londres")
+      end
+
+      crawler_instance = crawler.new
+
+      results = crawler_instance.crawl
+
+      results["search"].should == "Buscar"
+      results["links"].should == [{"menu"=>"Agenda"}, {"menu"=>"Brasileiro"}, {"menu"=>"Brasil"}, {"menu"=>"Bolsas"}, {"menu"=>"Cinema"}, {"menu"=>"Galerias de Fotos"}, {"menu"=>"Beleza"}, {"menu"=>"Esportes"}, {"menu"=>"Assine o RSS"}]
+      results["subheader"].should == "Londres 2012"
+      results["social"]["twitter"].should == "Verão"
+    end
+  end
+
   it 'should support hash based selectors' do
     VCR.use_cassette('basic_crawler_page') do
       crawler = Class.new
@@ -242,4 +273,4 @@ describe 'basic crawler setup' do
       }
     end
   end
-end
+end

data/spec/processing/parser_spec.rb

@@ -18,7 +18,7 @@ describe Wombat::Processing::Parser do
     fake_document.should_receive(:header).and_return(fake_header)
     fake_parser.should_receive(:headers=)
     @parser.mechanize.should_receive(:get).with("http://www.google.com/search").and_return fake_document
-    
+
     @parser.parse @metadata
   end
 
@@ -32,7 +32,20 @@ describe Wombat::Processing::Parser do
     Nokogiri.should_receive(:XML).with(fake_document).and_return fake_parser
     fake_document.should_receive(:headers).and_return(fake_headers)
     fake_parser.should_receive(:headers=)
-    
+
     @parser.parse @metadata
   end
-end
+
+  it 'should accept a Mechanize::Page' do
+    VCR.use_cassette('basic_crawler_page') do
+      m = Mechanize.new
+      page = m.get('http://www.terra.com.br/portal')
+      @metadata.page page
+
+      @parser.mechanize.should_not_receive(:get)
+
+      @parser.parse @metadata
+    end
+  end
+
+end

data/wombat.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "wombat"
-  s.version = "2.1.3"
+  s.version = "2.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Felipe Lima"]
-  s.date = "2013-04-22"
+  s.date = "2013-06-06"
   s.description = "Generic Web crawler with a DSL that parses structured data from web pages"
   s.email = "felipe.lima@gmail.com"
   s.extra_rdoc_files = [

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wombat
 version: !ruby/object:Gem::Version
-  version: 2.1.3
+  version: 2.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-22 00:00:00.000000000 Z
+date: 2013-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize