dubdubdub 0.1.0 → 0.1.1

data/Gemfile

@@ -1,6 +1,7 @@
 source "http://rubygems.org"
 
 gem 'rest-client'
+gem 'nokogiri'
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.

data/Gemfile.lock

@@ -13,6 +13,7 @@ GEM
     json (1.7.5)
     method_source (0.8.1)
     mime-types (1.19)
+    nokogiri (1.5.5)
     pry (0.9.10)
       coderay (~> 1.0.5)
       method_source (~> 0.8)
@@ -40,6 +41,7 @@ DEPENDENCIES
   bundler
   fakeweb (~> 1.3.0)
   jeweler (~> 1.8.4)
+  nokogiri
   pry
   rest-client
   rspec (~> 2.8.0)

data/dubdubdub.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = "dubdubdub"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["James Hu"]
@@ -30,6 +30,7 @@ Gem::Specification.new do |s|
     "spec/dubdubdub_spec.rb",
     "spec/spec_helper.rb",
     "spec/support/vcr.rb",
+    "spec/vcr/crawl/basic.yml",
     "spec/vcr/follow_url/alias_link.yml",
     "spec/vcr/follow_url/base.yml",
     "spec/vcr/follow_url/block_base_url.yml",
@@ -55,6 +56,7 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<rest-client>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
       s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
       s.add_development_dependency(%q<vcr>, ["~> 2.3.0"])
       s.add_development_dependency(%q<fakeweb>, ["~> 1.3.0"])
@@ -63,6 +65,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<pry>, [">= 0"])
     else
       s.add_dependency(%q<rest-client>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 0"])
       s.add_dependency(%q<rspec>, ["~> 2.8.0"])
       s.add_dependency(%q<vcr>, ["~> 2.3.0"])
       s.add_dependency(%q<fakeweb>, ["~> 1.3.0"])
@@ -72,6 +75,7 @@ Gem::Specification.new do |s|
     end
   else
     s.add_dependency(%q<rest-client>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 0"])
     s.add_dependency(%q<rspec>, ["~> 2.8.0"])
     s.add_dependency(%q<vcr>, ["~> 2.3.0"])
     s.add_dependency(%q<fakeweb>, ["~> 1.3.0"])

data/lib/dubdubdub.rb

@@ -1,6 +1,6 @@
 class DubDubDub
   # Version
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 
   attr_accessor :client
 

data/lib/dubdubdub/client.rb

@@ -1,5 +1,5 @@
-require 'open-uri'
 require 'net/http'
+require 'nokogiri'
 
 class DubDubDub::Client
   attr_accessor :proxy_host, :proxy_port, :proxy_user, :proxy_password
@@ -68,6 +68,13 @@ class DubDubDub::Client
     rest_client_resource(url).delete(*args)
   end
 
+  # Helper method to crawl by using a GET request
+  def crawl(url, *args)
+    response = get(url, *args)
+
+    Nokogiri::HTML(response.body)
+  end
+
   # Follow a url to the end until it can no longer go any further
   # Even if it times out, it will return the url that it times out on!
   def follow_url(url, options = {}, &block)

data/spec/dubdubdub_spec.rb

@@ -2,6 +2,7 @@
 
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 require 'rest-client'
+require 'nokogiri'
 
 describe DubDubDub do
   let(:www) { DubDubDub.new }
@@ -58,6 +59,13 @@ describe DubDubDub do
     end
   end
 
+  describe '#crawl' do
+    it "performs a GET request and returns a Nokogiri HTML document", vcr: { cassette_name: "crawl/basic", record: :once } do
+      html = www.crawl "http://www.google.com"
+      html.should be_a Nokogiri::HTML::Document
+    end
+  end
+
   describe '#follow_url' do
     it "follows url to the end", vcr: { cassette_name: "follow_url/base", record: :once } do
       www.follow_url("http://say.ly/TCc1CEp").should == "http://www.whosay.com/TomHanks/photos/148406"

data/spec/vcr/crawl/basic.yml

@@ -0,0 +1,169 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://www.google.com/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*; q=0.5, application/xml'
+      accept-encoding:
+      - gzip, deflate
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Mon, 03 Dec 2012 22:19:47 GMT
+      expires:
+      - '-1'
+      cache-control:
+      - private, max-age=0
+      content-type:
+      - text/html; charset=ISO-8859-1
+      set-cookie:
+      - PREF=ID=e141badc9e873db6:FF=0:TM=1354573187:LM=1354573187:S=LqBdtkOJGXGBU2Mt;
+        expires=Wed, 03-Dec-2014 22:19:47 GMT; path=/; domain=.google.com
+      - NID=66=rvfWllEd3r-ipw8tVDXDSneXoZaX497GlkHEg5mAQwiJ1G4FRVEHXBCxXxsooIg0yBiYR9GWBsbQml9tZGabIJ7BDk56eQk0k7qXR5xArBB_ECt-LrEEmxo4l6K7ZUeT;
+        expires=Tue, 04-Jun-2013 22:19:47 GMT; path=/; domain=.google.com; HttpOnly
+      p3p:
+      - CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657
+        for more info."
+      server:
+      - gws
+      x-xss-protection:
+      - 1; mode=block
+      x-frame-options:
+      - SAMEORIGIN
+      transfer-encoding:
+      - chunked
+    body:
+      encoding: US-ASCII
+      string: ! '<!doctype html><html itemscope="itemscope" itemtype="http://schema.org/WebPage"><head><meta
+        content="Search the world''s information, including webpages, images, videos
+        and more. Google has many special features to help you find exactly what you''re
+        looking for." name="description"><meta content="noodp" name="robots"><meta
+        itemprop="image" content="/images/google_favicon_128.png"><title>Google</title><script>(function(){
+
+        window.google={kEI:"gyW9UJ2LDsHuigKL5ICgDw",getEI:function(a){for(var b;a&&(!a.getAttribute||!(b=a.getAttribute("eid")));)a=a.parentNode;return
+        b||google.kEI},https:function(){return"https:"==window.location.protocol},kEXPI:"17259,24692,24878,24879,27400,35703,35705,36726,39523,39978,40363,4000116,4000473,4000945,4000955,4001372,4001456,4001569,4001855,4001933,4001966,4002000,4002036,4002161,4002240,4002274,4002348,4002359,4002378,4002391,4002436,4002460,4002466,4002510,4002562,4002710,4002733,4002743,4002771,4002788",kCSI:{e:"17259,24692,24878,24879,27400,35703,35705,36726,39523,39978,40363,4000116,4000473,4000945,4000955,4001372,4001456,4001569,4001855,4001933,4001966,4002000,4002036,4002161,4002240,4002274,4002348,4002359,4002378,4002391,4002436,4002460,4002466,4002510,4002562,4002710,4002733,4002743,4002771,4002788",ei:"gyW9UJ2LDsHuigKL5ICgDw"},authuser:0,ml:function(){},kHL:"en",time:function(){return(new
+        Date).getTime()},log:function(a,
+
+        b,c,j){var d=new Image,f=google.lc,e=google.li,g="";d.onerror=d.onload=d.onabort=function(){delete
+        f[e]};f[e]=d;!c&&-1==b.search("&ei=")&&(g="&ei="+google.getEI(j));c=c||"/gen_204?atyp=i&ct="+a+"&cad="+b+g+"&zx="+google.time();a=/^http:/i;a.test(c)&&google.https()?(google.ml(Error("GLMM"),!1,{src:c}),delete
+        f[e]):(d.src=c,google.li=e+1)},lc:[],li:0,Toolbelt:{},y:{},x:function(a,b){google.y[a.id]=[a,b];return!1}};})();
+
+        window.google.sn="webhp";window.google.timers={};window.google.startTick=function(a,b){window.google.timers[a]={t:{start:(new
+        Date).getTime()},bfr:!(!b)}};window.google.tick=function(a,b,c){if(!window.google.timers[a])google.startTick(a);window.google.timers[a].t[b]=c||(new
+        Date).getTime()};google.startTick("load",true);try{}catch(u){}
+
+        var _gjwl=location;function _gjuc(){var a=_gjwl.href.indexOf("#");if(0<=a&&(a=_gjwl.href.substring(a),0<a.indexOf("&q=")||0<=a.indexOf("#q=")))if(a=a.substring(1),-1==a.indexOf("#")){for(var
+        d=0;d<a.length;){var b=d;"&"==a.charAt(b)&&++b;var c=a.indexOf("&",b);-1==c&&(c=a.length);b=a.substring(b,c);if(0==b.indexOf("fp="))a=a.substring(0,d)+a.substring(c,a.length),c=d;else
+        if("cad=h"==b)return 0;d=c}_gjwl.href="/search?"+a+"&cad=h";return 1}return
+        0}
+
+        function _gjp(){(!window._gjwl.hash||!window._gjuc())&&setTimeout(_gjp,500)};
+
+        window._gjp&&_gjp();</script><style>#gbar,#guser{font-size:13px;padding-top:1px
+        !important;}#gbar{height:22px}#guser{padding-bottom:7px !important;text-align:right}.gbh,.gbd{border-top:1px
+        solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}@media
+        all{.gb1{height:22;margin-right:.5em;vertical-align:top}#gbar{float:left}}a.gb1,a.gb4{text-decoration:underline
+        !important}a.gb1,a.gb4{color:#00c !important}.gbi .gb4{color:#dd8e27 !important}.gbf
+        .gb4{color:#900 !important}</style><style id="gstyle">body{margin:0;overflow-y:scroll}#gog{padding:3px
+        8px 0}td{line-height:.8em}.gac_m td{line-height:17px}form{margin-bottom:20px}body,td,a,p,.h{font-family:arial,sans-serif}.h{color:#36c;font-size:20px}.q{color:#00c}.ts
+        td{padding:0}.ts{border-collapse:collapse}em{font-weight:bold;font-style:normal}.lst{height:25px;width:496px}.gsfi,.lst{font:18px
+        arial,sans-serif}.gsfs{font:17px arial,sans-serif}.ds{display:-moz-inline-box;display:inline-block;margin:3px
+        0 4px;margin-left:4px}input{font-family:inherit}a.gb1,a.gb2,a.gb3,a.gb4{color:#11c
+        !important}body{background:#fff;color:black}a{color:#11c;text-decoration:none}a:hover,a:active{text-decoration:underline}.fl
+        a{color:#36c}a:visited{color:#551a8b}a.gb1,a.gb4{text-decoration:underline}a.gb3:hover{text-decoration:none}#ghead
+        a.gb2:hover{color:#fff!important}.sblc{padding-top:5px}.sblc a{display:block;margin:2px
+        0;margin-left:13px;font-size:11px;}.lsbb{background:#eee;border:solid 1px;border-color:#ccc
+        #999 #999 #ccc;height:30px;display:block}.ftl,#fll a{display:inline-block;margin:0
+        12px}.lsb{background:url(/images/srpr/nav_logo80.png) 0 -258px repeat-x;border:none;color:#000;cursor:pointer;height:30px;margin:0;outline:0;font:15px
+        arial,sans-serif;vertical-align:top}.lsb:active{background:#ccc}.lst:focus{outline:none}#addlang
+        a{padding:0 3px}.gac_v div{display:none}.gac_v .gac_v2,.gac_bt{display:block!important}table.gssb_c{z-index:986  }.nbcl{background:url(/images/srpr/nav_logo80.png)
+        no-repeat ;height:px;width:px}</style><script></script> </head><body dir="ltr"
+        bgcolor="#fff"><script>(function(){var src=''/images/srpr/nav_logo80.png'';var
+        iesg=false;document.body.onload = function(){window.n && window.n();if (document.images){new
+        Image().src=src;}
+
+        if (!iesg){document.f&&document.f.q.focus();document.gbqf&&document.gbqf.q.focus();}
+
+        }
+
+        })();</script><textarea id="csi" style="display:none"></textarea><div id="mngb"><div
+        id=gbar><nobr><b class=gb1>Search</b> <a class=gb1 href="http://www.google.com/imghp?hl=en&tab=wi">Images</a>
+        <a class=gb1 href="http://maps.google.com/maps?hl=en&tab=wl">Maps</a> <a class=gb1
+        href="https://play.google.com/?hl=en&tab=w8">Play</a> <a class=gb1 href="http://www.youtube.com/?tab=wy">YouTube</a>
+        <a class=gb1 href="http://news.google.com/nwshp?hl=en&tab=wn">News</a> <a
+        class=gb1 href="https://mail.google.com/mail/?tab=wm">Gmail</a> <a class=gb1
+        href="https://drive.google.com/?tab=wo">Drive</a> <a class=gb1 style="text-decoration:none"
+        href="http://www.google.com/intl/en/options/"><u>More</u> &raquo;</a></nobr></div><div
+        id=guser width=100%><nobr><span id=gbn class=gbi></span><span id=gbf class=gbf></span><span
+        id=gbe></span><a href="http://www.google.com/history/optout?hl=en" class=gb4>Web
+        History</a> | <a  href="/preferences?hl=en" class=gb4>Settings</a> | <a target=_top
+        id=gb_70 href="https://accounts.google.com/ServiceLogin?hl=en&continue=http://www.google.com/"
+        class=gb4>Sign in</a></nobr></div><div class=gbh style=left:0></div><div class=gbh
+        style=right:0></div></div><iframe name="wgjf" style="display:none"></iframe><center><br
+        clear="all" id="lgpd"><div id="lga"><img alt="Google" height="95" src="/intl/en_ALL/images/srpr/logo1w.png"
+        width="275" id="hplogo" onload="window.lol&&lol()" style="padding:28px 0 14px"><br><br></div><form
+        action="/search" name="f"><table cellpadding="0" cellspacing="0"><tr valign="top"><td
+        width="25%">&nbsp;</td><td align="center" nowrap="nowrap"><input name="ie"
+        value="ISO-8859-1" type="hidden"><input value="en" name="hl" type="hidden"><input
+        name="source" type="hidden" value="hp"><div class="ds" style="height:32px;margin:4px
+        0"><input autocomplete="off" class="lst" value="" title="Google Search" maxlength="2048"
+        name="q" size="57" style="color:#000;margin:0;padding:5px 8px 0 6px;vertical-align:top"></div><br
+        style="line-height:0"><span class="ds"><span class="lsbb"><input class="lsb"
+        value="Google Search" name="btnG" type="submit"></span></span><span class="ds"><span
+        class="lsbb"><input class="lsb" value="I''m Feeling Lucky" name="btnI" type="submit"
+        onclick="if(this.form.q.value)this.checked=1; else top.location=''/doodles/''"></span></span></td><td
+        class="fl sblc" align="left" nowrap="nowrap" width="25%"><a href="/advanced_search?hl=en&amp;authuser=0">Advanced
+        search</a><a href="/language_tools?hl=en&amp;authuser=0">Language tools</a></td></tr></table><input
+        type="hidden" id="gbv" name="gbv" value="1"></form><div id="gac_scont"></div><div
+        style="font-size:83%;min-height:3.5em"><br><div id="prm"><font id="hpplink"
+        size="-1" style="behavior:url(#default#userdata)"><span style="color:red"></span>Love
+        the free and open Internet? <a href="https://www.google.com/takeaction/?utm_source=google&amp;utm_medium=hpp&amp;utm_campaign=12032012freeandopen_en"
+        onclick="google.promos&&google.promos.link&& google.promos.link.cl()">Tell
+        the world''s governments</a> to keep it that way.</font><br><br><br><script
+        type="text/javascript">(function(){var d={promos:{}};d.promos.localStorage={};d.promos.localStorage.isLocalstorageSupported=function(){try{return"object"==typeof
+        window.localStorage}catch(a){return!1}};d.promos.localStorage.getPersistData=function(a){a=window.localStorage.getItem(a);return!a?"":a.toString()};d.promos.localStorage.setPersistData=function(a,b){window.localStorage.setItem(a,b)};d.promos.localStorage.removePersistData=function(a){window.localStorage.removeItem(a)};google.promos||(google.promos={});google.promos.ActionType={ACCEPT:"a",CANCEL:"c",DISMISS:"d",CLICK:"h",IMPRESSION:"i",NO_THANKS:"n",X_BUTTON:"x",MGMHP_ACCEPT:"ma",MGMHP_CANCEL:"mc",MGMHP_IMPRESSION:"mi",MGMHPPD_ACCEPT:"pa",MGMHPPD_CANCEL:"pc",MGMHPPD_IMPRESSION:"pi",MGMHPPD_NO_THANKS:"pn",MGMHPPD_NO_BUTTON:"px",MGMHPPD_DISMISS:"pd",PUSHDOWN_ACCEPT:"gpa",PUSHDOWN_IMPRESSION:"gpi",PUSHDOWN_NO_THANKS:"gpn",PUSHDOWN_NO_BUTTON:"gpx",PUSHDOWN_DISMISS:"gpd"};google.promos.sl=function(a,b,c,e){a=[c,"id="+a,"loc="+google.sn];a.push("oi=promoslinger");e&&a.push(e);google.log(b,a.join("&"))};google.promos.si=function(a,b,c,e){0.01>Math.random()&&google.promos.sl(a,b,e?e:google.promos.ActionType.IMPRESSION,c)};google.promos.spd=function(a,b,c){if(d.promos.localStorage.isLocalstorageSupported())d.promos.localStorage.setPersistData(b,c);else
+        if(a)try{a.setAttribute(b,c),a.save(a.id)}catch(e){google.ml(e,!1,{cause:"PERSIST_DATA_FAIL"})}};google.promos.gpd=function(a,b){if(d.promos.localStorage.isLocalstorageSupported())return
+        d.promos.localStorage.getPersistData(b);if(a)try{return a.load(a.id),a.getAttribute(b)}catch(c){google.ml(c,!1,{cause:"PERSIST_DATA_FAIL"})}return""};google.promos.rpd=function(a,b){if(d.promos.localStorage.isLocalstorageSupported())d.promos.localStorage.removePersistData(b);else
+        if(a)try{a.load(a.id),a.removeAttribute(b),a.save(a.id)}catch(c){google.ml(c,!1,{cause:"PERSIST_DATA_FAIL"})}};google.promos.aeh=function(a,b,c){a.addEventListener?a.addEventListener(b,c,!1):a.attachEvent&&a.attachEvent("on"+b,c)};})();</script><script
+        type="text/javascript">(function(){var c,e,g=0,h;google.promos.link||(google.promos.link={});google.promos.link.getExtraLogData_=function(b){var
+        a=document.getElementById(h);if(a){var f=e+"_upccb",d=parseInt(google.promos.gpd(a,f)||0,10);d++;g++;google.promos.spd(a,f,d);a=[["upcc",g].join("="),["upccb",d].join("=")];b&&a.push(b);return
+        a.join("&")}};google.promos.link.cl=function(b){try{google.promos.sl(e,c,google.promos.ActionType.CLICK,google.promos.link.getExtraLogData_(b))}catch(a){google.ml(a,!1,{cause:c+"_CL"})}};google.promos.link.init=function(b,a,f){try{e=b,c=a,h=f,google.promos.si(e,c)}catch(d){google.ml(d,!1,{cause:c+"_INIT"})}};})();</script><script>(function(){var
+        sourceWebappPromoID=1579001;var payloadType=3;google.promos.link.init(sourceWebappPromoID,payloadType,''hpplink'');})();</script></div></div><span
+        id="footer"><div style="font-size:10pt"><div id="fll" style="margin:19px auto;text-align:center"><a
+        href="/intl/en/ads/">Advertising&nbsp;Programs</a><a href="/services/">Business
+        Solutions</a><a href="https://plus.google.com/116899029375914044550" rel="publisher">+Google</a><a
+        href="/intl/en/about.html">About Google</a></div></div><p style="color:#767676;font-size:8pt">&copy;
+        2012 - <a href="/intl/en/policies/">Privacy & Terms</a></p></span></center><div
+        id=xjsd></div><div id=xjsi><script>if(google.y)google.y.first=[];(function(){var
+        b;function c(a){window.setTimeout(function(){var d=document.createElement("script");d.src=a;document.getElementById("xjsd").appendChild(d)},0)}google.dljp=function(a){b=a;google.xjsi||(google.xjsu=a,c(b))};google.dlj=c;})();
+
+        if(!google.xjs){google.dstr=[];google.rein=[];window._=window._||{};window._._DumpException=function(e){throw
+        e};if(google.timers&&google.timers.load.t){google.timers.load.t.xjsls=new
+        Date().getTime();}google.dljp(''/xjs/_/js/hp/sb_he,pcc/rt\x3dj/ver\x3dekMxmVjFIzI.en_US./d\x3d1/sv\x3d1/rs\x3dAItRSTM0lksRtI2VxMWJM1mGGSAc7V2OnQ'');google.xjs=1;}google.pmc={sb:{"agen":false,"cgen":true,"client":"heirloom-hp","dh":true,"ds":"","eqch":true,"fl":true,"host":"google.com","jsonp":true,"msgs":{"lcky":"I\u0026#39;m
+        Feeling Lucky","lml":"Learn more","oskt":"Input tools","psrc":"This search
+        was removed from your \u003Ca href=\"/history\"\u003EWeb History\u003C/a\u003E","psrl":"Remove","sbit":"Search
+        by image","srch":"Google Search"},"ovr":{"l":1,"ms":1},"pq":"","qcpw":false,"scd":10,"sce":5,"stok":"T6XUP3D9Euph7X6BWXXXesAOH9Q"},pcc:{}};google.y.first.push(function(){if(google.med){google.med(''init'');google.initHistory();google.med(''history'');}google.History&&google.History.initialize(''/'');google.hs&&google.hs.init&&google.hs.init()});if(google.j&&google.j.en&&google.j.xi){window.setTimeout(google.j.xi,0);}</script></div><script>(function(){var
+        b,c,d,e;function g(a,f){a.removeEventListener?(a.removeEventListener("load",f,!1),a.removeEventListener("error",f,!1)):(a.detachEvent("onload",f),a.detachEvent("onerror",f))}function
+        h(a){e=(new Date).getTime();++c;a=a||window.event;a=a.target||a.srcElement;g(a,h)}var
+        j=document.getElementsByTagName("img");b=j.length;
+
+        for(var k=c=0,l;k<b;++k)l=j[k],l.complete||"string"!=typeof l.src||!l.src?++c:l.addEventListener?(l.addEventListener("load",h,!1),l.addEventListener("error",h,!1)):(l.attachEvent("onload",h),l.attachEvent("onerror",h));d=b-c;
+
+        function m(){if(google.timers.load.t){google.timers.load.t.ol=(new Date).getTime();google.timers.load.t.iml=e;google.kCSI.imc=c;google.kCSI.imn=b;google.kCSI.imp=d;void
+        0!==google.stt&&(google.kCSI.stt=google.stt);google.csiReport&&google.csiReport()}}window.addEventListener?window.addEventListener("load",m,!1):window.attachEvent&&window.attachEvent("onload",m);google.timers.load.t.prt=e=(new
+        Date).getTime();})();
+
+        </script></body></html>'
+    http_version: '1.1'
+  recorded_at: Mon, 03 Dec 2012 22:19:47 GMT
+recorded_with: VCR 2.3.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dubdubdub
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -144,6 +160,7 @@ files:
 - spec/dubdubdub_spec.rb
 - spec/spec_helper.rb
 - spec/support/vcr.rb
+- spec/vcr/crawl/basic.yml
 - spec/vcr/follow_url/alias_link.yml
 - spec/vcr/follow_url/base.yml
 - spec/vcr/follow_url/block_base_url.yml
@@ -172,7 +189,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2986985482614127435
+      hash: 1241475245500893694
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: