wombat 2.5.0 → 2.5.1

data/lib/wombat/processing/parser.rb

@@ -9,6 +9,9 @@ module Nokogiri
     class Document
       attr_accessor :headers
     end
+    class Element
+      attr_accessor :mechanize_page
+    end
   end
   module HTML
     class Document

data/spec/integration/integration_spec.rb

@@ -274,6 +274,130 @@ describe 'basic crawler setup' do
     end
   end
 
+  it 'should follow links - issue #53' do
+    VCR.use_cassette('follow_links_v2', :preserve_exact_body_bytes => true) do
+      result = Wombat.crawl do
+        base_url "http://www.icy-veins.com/"
+        path "heroes/hero-guides"
+
+        heroes  "css=.page_content .nav_content_block_entry_heroes_hero", :iterator do
+          name "xpath=."
+          builds "xpath=./a", :follow do
+              title "css=h1"
+          end
+        end
+      end
+
+      expect(result).to eq(
+        {"heroes"=>
+          [{"name"=>"Abathur",
+            "builds"=>[{"title"=>"Abathur Build Guide “You. Enhanced. Improved.”"}]},
+           {"name"=>"Anub'arak",
+            "builds"=>[{"title"=>"Anub'arak Build Guide “Time is fleeting.”"}]},
+           {"name"=>"Artanis",
+            "builds"=>[{"title"=>"Artanis Build Guide “Direct my wrath.”"}]},
+           {"name"=>"Arthas",
+            "builds"=>[{"title"=>"Arthas Build Guide “Frostmourne hungers.”"}]},
+           {"name"=>"Azmodan",
+            "builds"=>[{"title"=>"Azmodan Build Guide “I shall rule alone!”"}]},
+           {"name"=>"Brightwing",
+            "builds"=>
+             [{"title"=>"Brightwing Build Guide “You don't want to be my enemy!”"}]},
+           {"name"=>"Chen",
+            "builds"=>[{"title"=>"Chen Build Guide “I bring Pandamonium!”"}]},
+           {"name"=>"Cho",
+            "builds"=>
+             [{"title"=>
+                "Cho Build Guide “This Nexus, all its power... it will be MINE!”"}]},
+           {"name"=>"Diablo",
+            "builds"=>
+             [{"title"=>"Diablo Build Guide “Kneel before the Lord of Terror.”"}]},
+           {"name"=>"E.T.C.",
+            "builds"=>[{"title"=>"E.T.C. Build Guide “You can't kill the metal!”"}]},
+           {"name"=>"Falstad",
+            "builds"=>[{"title"=>"Falstad Build Guide “Time to drop the hammer!”"}]},
+           {"name"=>"Gall",
+            "builds"=>
+             [{"title"=>
+                "Gall Build Guide “Those who oppose me invite their own demise!”"}]},
+           {"name"=>"Gazlowe",
+            "builds"=>[{"title"=>"Gazlowe Build Guide “Hey, time is money friend.”"}]},
+           {"name"=>"Greymane",
+            "builds"=>[{"title"=>"Greymane Build Guide “I am the alpha!”"}]},
+           {"name"=>"Illidan",
+            "builds"=>[{"title"=>"Illidan Build Guide “Now I am complete!”"}]},
+           {"name"=>"Jaina",
+            "builds"=>[{"title"=>"Jaina Build Guide “I'm here to help.”"}]},
+           {"name"=>"Johanna",
+            "builds"=>[{"title"=>"Johanna Build Guide “The Crusade marches on!”"}]},
+           {"name"=>"Kael'thas",
+            "builds"=>[{"title"=>"Kael'thas Build Guide “Anar'alah belore!”"}]},
+           {"name"=>"Kerrigan",
+            "builds"=>[{"title"=>"Kerrigan Build Guide “Long live the real Queen.”"}]},
+           {"name"=>"Kharazim",
+            "builds"=>[{"title"=>"Kharazim Build Guide “Feel the wrath of Ytar!”"}]},
+           {"name"=>"Leoric",
+            "builds"=>
+             [{"title"=>"Leoric Build Guide “All will suffer as I have suffered!”"}]},
+           {"name"=>"Li Li",
+            "builds"=>[{"title"=>"Li Li Build Guide “Ready for adventure!”"}]},
+           {"name"=>"Lt. Morales",
+            "builds"=>
+             [{"title"=>
+                "Lt. Morales Build Guide “I protect every member of my squad!”"}]},
+           {"name"=>"Lunara",
+            "builds"=>[{"title"=>"Lunara Build Guide “Taste my spear!”"}]},
+           {"name"=>"Malfurion",
+            "builds"=>
+             [{"title"=>"Malfurion Build Guide “Nature will rise against you!”"}]},
+           {"name"=>"Muradin",
+            "builds"=>[{"title"=>"Muradin Build Guide “It's hammer time!”"}]},
+           {"name"=>"Murky",
+            "builds"=>[{"title"=>"Murky Build Guide “Mrglrglmrglmrrrlggg!”"}]},
+           {"name"=>"Nazeebo",
+            "builds"=>[{"title"=>"Nazeebo Build Guide “The spirits speak to me.”"}]},
+           {"name"=>"Nova",
+            "builds"=>
+             [{"title"=>"Nova Build Guide “Ready to have your mind blown?”"}]},
+           {"name"=>"Raynor",
+            "builds"=>[{"title"=>"Raynor Build Guide “Hit 'em hard and fast.”"}]},
+           {"name"=>"Rehgar",
+            "builds"=>[{"title"=>"Rehgar Build Guide “To the Arena!”"}]},
+           {"name"=>"Rexxar",
+            "builds"=>[{"title"=>"Rexxar Build Guide “The beasts obey me!”"}]},
+           {"name"=>"Sgt. Hammer",
+            "builds"=>[{"title"=>"Sgt. Hammer Build Guide “Napalm's airborne!”"}]},
+           {"name"=>"Sonya",
+            "builds"=>[{"title"=>"Sonya Build Guide “Time to die!”"}]},
+           {"name"=>"Stitches",
+            "builds"=>[{"title"=>"Stitches Build Guide “ROAAAARR!”"}]},
+           {"name"=>"Sylvanas",
+            "builds"=>[{"title"=>"Sylvanas Build Guide “Let none survive!”"}]},
+           {"name"=>"Tassadar",
+            "builds"=>[{"title"=>"Tassadar Build Guide “Executor, I stand ready!”"}]},
+           {"name"=>"The Butcher",
+            "builds"=>[{"title"=>"The Butcher Build Guide “Fresh meat!”"}]},
+           {"name"=>"Thrall",
+            "builds"=>
+             [{"title"=>"Thrall Build Guide “The Elements will destroy you!”"}]},
+           {"name"=>"Tychus",
+            "builds"=>[{"title"=>"Tychus Build Guide “So, you gonna bark all day?”"}]},
+           {"name"=>"Tyrael",
+            "builds"=>[{"title"=>"Tyrael Build Guide “I am Justice itself!”"}]},
+           {"name"=>"Tyrande",
+            "builds"=>
+             [{"title"=>"Tyrande Build Guide “Feel the wrath of the Heavens!”"}]},
+           {"name"=>"Uther",
+            "builds"=>[{"title"=>"Uther Build Guide “I will fight with honor!”"}]},
+           {"name"=>"Valla",
+            "builds"=>[{"title"=>"Valla Build Guide “Be vewy, vewy quiet...”"}]},
+           {"name"=>"Zagara",
+            "builds"=>[{"title"=>"Zagara Build Guide “The Swarm hungers.”"}]},
+           {"name"=>"Zeratul",
+            "builds"=>[{"title"=>"Zeratul Build Guide “I serve the Xel'naga.”"}]}]})
+    end
+  end
+
   it 'should make post requests if needed' do
     VCR.use_cassette('make_post_request') do
       data = { your_name: "Name" }

data/spec/sample_crawler_spec.rb

@@ -10,13 +10,14 @@ describe SampleCrawler do
     @sample_crawler.should_receive(:parse) do |args|
       args['event_group'].wombat_property_selector.should == "css=div.title-agenda"
       it = args['event_group']
-      it["event"]["title"].wombat_property_selector.should == "xpath=."
-      it["event"]["date"].wombat_property_selector.should == "xpath=//div[@class='scrollable-items']/div[@class='s-item active']//a"
-      it["event"]["type"].wombat_property_selector.should == "xpath=.type"
-      it["venue"]["name"].wombat_property_selector.should == "xpath=."
+      expect(it["event"]["title"].wombat_property_selector).to eq("xpath=.")
+      expect(it["event"]["date"].wombat_property_selector).to(
+        eq("xpath=//div[@class='scrollable-items']/div[@class='s-item active']//a"))
+      expect(it["event"]["type"].wombat_property_selector).to eq("xpath=.type")
+      expect(it["venue"]["name"].wombat_property_selector).to eq("xpath=.")
 
       args[:base_url].should == 'http://www.obaoba.com.br'
-      args[:path].should == '/porto-alegre/agenda' 
+      args[:path].should == '/porto-alegre/agenda'
     end
 
     @sample_crawler.crawl

data/wombat.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: wombat 2.5.0 ruby lib
+# stub: wombat 2.5.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "wombat"
-  s.version = "2.5.0"
+  s.version = "2.5.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Felipe Lima"]
-  s.date = "2016-01-27"
+  s.date = "2016-03-31"
   s.description = "Generic Web crawler with a DSL that parses structured data from web pages"
   s.email = "felipe.lima@gmail.com"
   s.extra_rdoc_files = [
@@ -40,6 +40,7 @@ Gem::Specification.new do |s|
     "fixtures/vcr_cassettes/broken_selector.yml",
     "fixtures/vcr_cassettes/error_page.yml",
     "fixtures/vcr_cassettes/follow_links.yml",
+    "fixtures/vcr_cassettes/follow_links_v2.yml",
     "fixtures/vcr_cassettes/follow_relative_links.yml",
     "fixtures/vcr_cassettes/for_each_page.yml",
     "fixtures/vcr_cassettes/headers_selector.yml",
@@ -86,7 +87,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://felipecsl.github.com/wombat"
   s.licenses = ["MIT"]
   s.required_ruby_version = Gem::Requirement.new(">= 1.9")
-  s.rubygems_version = "2.4.6"
+  s.rubygems_version = "2.4.8"
   s.summary = "Ruby DSL to scrape web pages"
 
   if s.respond_to? :specification_version then

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wombat
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.5.1
 platform: ruby
 authors:
 - Felipe Lima
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-27 00:00:00.000000000 Z
+date: 2016-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -193,6 +193,7 @@ files:
 - fixtures/vcr_cassettes/broken_selector.yml
 - fixtures/vcr_cassettes/error_page.yml
 - fixtures/vcr_cassettes/follow_links.yml
+- fixtures/vcr_cassettes/follow_links_v2.yml
 - fixtures/vcr_cassettes/follow_relative_links.yml
 - fixtures/vcr_cassettes/for_each_page.yml
 - fixtures/vcr_cassettes/headers_selector.yml
@@ -255,7 +256,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Ruby DSL to scrape web pages