wombat 2.2.1 → 2.3.0

data/lib/wombat/processing/parser.rb

@@ -18,7 +18,14 @@ module Wombat
       attr_accessor :mechanize, :context, :response_code, :page
 
       def initialize
-        @mechanize = Mechanize.new
+        # http://stackoverflow.com/questions/6918277/ruby-mechanize-web-scraper-library-returns-file-instead-of-page
+        @mechanize = Mechanize.new { |a|
+          a.post_connect_hooks << lambda { |_,_,response,_|
+            if response.content_type.nil? || response.content_type.empty?
+              response.content_type = 'text/html'
+            end
+          }
+        }
         @mechanize.set_proxy(*Wombat.proxy_args) if Wombat.proxy_args
       end
 

data/lib/wombat/property/locators/follow.rb

@@ -7,14 +7,40 @@ module Wombat
 	    	def locate(context, page = nil)
 	    		super do
             locate_nodes(context).flat_map do |node|
-              target_page = page.click node
-              context = target_page.parser
+              retried = false
+              begin
+                # Certain erroneous pages contain http 
+                # links with relative href attribute, 
+                # while browsers actually use them as 
+                # absolute.
+                # So, let wombat try that approach when 
+                # loading relative link fails.
+                #
+                target_page = page.click node
+                context = target_page.parser
 
-              filter_properties(context, page)
+                filter_properties(context, page)
+              rescue Mechanize::ResponseCodeError => e
+                # Either the page is unavailable, or
+                # the link is mistakenly relative
+                #
+                raise e if retried
+                
+                # Give it a try first time
+                href = node.attributes && node.attributes["href"]
+                if href.respond_to? :value
+                  href.value = '/' + href.value unless 
+                    href.value.start_with? '/'
+                  retried = true
+                  retry
+                else
+                  raise e
+                end
+              end
             end
           end
 	    	end
 	    end
 	  end
 	end
-end
+end

data/spec/crawler_spec.rb

@@ -13,7 +13,7 @@ describe Wombat::Crawler do
 
       @crawler.event { event_called = true }
 
-      event_called.should be_true
+      event_called.should eq(true)
     end
 
     it 'should provide metadata to yielded block' do
@@ -30,7 +30,7 @@ describe Wombat::Crawler do
         e.time Time.now
       end
 
-      @crawler.venue do |v| 
+      @crawler.venue do |v|
         v.name "Scooba"
       end
 

data/spec/dsl/property_spec.rb

@@ -2,11 +2,12 @@ require 'spec_helper'
 
 describe Wombat::DSL::Property do
   it 'should store property data' do
-    property = Wombat::DSL::Property.new("title", *["/some/selector", :html]) { false }
+    callback = lambda { false }
+    property = Wombat::DSL::Property.new("title", *["/some/selector", :html], &callback)
     
     property.wombat_property_name.should == "title"
     property.selector.should == "/some/selector"
     property.format.should == :html
-    property.callback.should == lambda { false }
+    property.callback.should == callback
   end
-end
+end

data/spec/integration/follow_relative_links_spec.rb

@@ -0,0 +1,33 @@
+# coding: utf-8
+require 'spec_helper'
+
+describe 'following pages referred by relative links' do
+  it 'should follow relative links' do
+    VCR.use_cassette('follow_relative_links') do
+      crawler = Class.new
+      crawler.send(:include, Wombat::Crawler)
+
+      crawler.base_url "http://liteproblog.ru/"
+      crawler.path '/vocabulary'
+
+      crawler.vocabulary 'css=.postcontent ul li a', :follow do
+        entry do
+          word 'css=.post p strong', :text
+          description 'css=.post p'
+        end
+      end
+
+      crawler_instance = crawler.new
+
+      results = crawler_instance.crawl
+
+      # There are many entries. It's enough to check first three ones
+      results["vocabulary"][0..2].should == [
+{"entry"=>{"word"=>"Dmoz", "description"=>"Dmoz - второй по популярности каталог сайтов после Яндекс-Каталога. Адрес каталога Dmoz - .\r\n\r\nЗаметка: Как вы думаете, мебель из Китая дорого стоит? Правильно, она недорогая.  поставляет не только мебель, но и китайскую сантехнику, люстры, светильники и многое другое. Если вы хотите здорово съэкономить, то не пропустите такую возможность."}}, 
+{"entry"=>{"word"=>"PR", "description"=>"PR - PageRank - показатель Google для конкретной страницы сайта. Зависит от количества ссылок на страницу и от качества этих ссылок. Учитываются и ссылки с внутренних страниц сайта. PR влияет на выдачу в поисковой системе Google. Повысить PR сайту можно внутренней перелинковкой. PR бывает тулбарный и внутренний. Апдейт PR происходит, как правило, несколько раз в год. Сейчас у этого блога PR=2, а у сайта  PR равен 3."}}, 
+{"entry"=>{"word"=>"Sape (сапа)", "description"=>"Sape (сапа) - это самая популярная в России биржа ссылок. Адрес: www.sape.ru. Веб-мастер может продать ссылки со своего сайта, а оптимизатор купить ссылки. Продажа ссылок осуществляется с ежемесячной оплатой. Цена на ссылки устанавливается веб-мастером для своего сайта. Для продажи ссылок на сайте размещается специальный код системы и в дальнейшем вся продажа происходит автоматически через веб-интерфейс Sape.\r\n\r\nЗаметка: Интересует монтаж и эксплуатация противопожарных металлических ДПМ или ? Читайте технологическую документацию и нормативные документы."}}
+      ]
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -1,6 +1,5 @@
 require 'wombat'
 require 'rspec'
-require 'rspec-expectations'
 require 'vcr'
 require 'coveralls'
 

data/wombat.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: wombat 2.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "wombat"
-  s.version = "2.2.1"
+  s.version = "2.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Felipe Lima"]
-  s.date = "2013-10-20"
+  s.date = "2014-12-02"
   s.description = "Generic Web crawler with a DSL that parses structured data from web pages"
   s.email = "felipe.lima@gmail.com"
   s.extra_rdoc_files = [
@@ -36,6 +38,7 @@ Gem::Specification.new do |s|
     "fixtures/vcr_cassettes/broken_selector.yml",
     "fixtures/vcr_cassettes/error_page.yml",
     "fixtures/vcr_cassettes/follow_links.yml",
+    "fixtures/vcr_cassettes/follow_relative_links.yml",
     "fixtures/vcr_cassettes/for_each_page.yml",
     "fixtures/vcr_cassettes/headers_selector.yml",
     "fixtures/vcr_cassettes/xml_with_namespace.yml",
@@ -61,6 +64,7 @@ Gem::Specification.new do |s|
     "spec/crawler_spec.rb",
     "spec/dsl/property_spec.rb",
     "spec/helpers/sample_crawler.rb",
+    "spec/integration/follow_relative_links_spec.rb",
     "spec/integration/integration_spec.rb",
     "spec/processing/parser_spec.rb",
     "spec/property/locators/factory_spec.rb",
@@ -77,13 +81,12 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://felipecsl.github.com/wombat"
   s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
   s.required_ruby_version = Gem::Requirement.new(">= 1.9")
-  s.rubygems_version = "1.8.24"
+  s.rubygems_version = "2.2.2"
   s.summary = "Ruby DSL to scrape web pages"
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<mechanize>, [">= 0"])

metadata

@@ -1,190 +1,167 @@
 --- !ruby/object:Gem::Specification
 name: wombat
 version: !ruby/object:Gem::Version
-  version: 2.2.1
-  prerelease: 
+  version: 2.3.0
 platform: ruby
 authors:
 - Felipe Lima
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-20 00:00:00.000000000 Z
+date: 2014-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fakeweb
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Generic Web crawler with a DSL that parses structured data from web pages
@@ -195,9 +172,9 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.md
 files:
-- .document
-- .rspec
-- .travis.yml
+- ".document"
+- ".rspec"
+- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - Guardfile
@@ -214,6 +191,7 @@ files:
 - fixtures/vcr_cassettes/broken_selector.yml
 - fixtures/vcr_cassettes/error_page.yml
 - fixtures/vcr_cassettes/follow_links.yml
+- fixtures/vcr_cassettes/follow_relative_links.yml
 - fixtures/vcr_cassettes/for_each_page.yml
 - fixtures/vcr_cassettes/headers_selector.yml
 - fixtures/vcr_cassettes/xml_with_namespace.yml
@@ -239,6 +217,7 @@ files:
 - spec/crawler_spec.rb
 - spec/dsl/property_spec.rb
 - spec/helpers/sample_crawler.rb
+- spec/integration/follow_relative_links_spec.rb
 - spec/integration/integration_spec.rb
 - spec/processing/parser_spec.rb
 - spec/property/locators/factory_spec.rb
@@ -255,26 +234,25 @@ files:
 homepage: http://felipecsl.github.com/wombat
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Ruby DSL to scrape web pages
 test_files: []