cobweb 0.0.32 → 0.0.33

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.32
+h1. Cobweb v0.0.33
 
 h2. Intro 
   
@@ -95,13 +95,9 @@ Simple get that obey's the options supplied in new.
 bq. crawler.head("http://www.google.com/")
 
 
-h2. Planned Work
+h3. Contributing/Testing
 
-These features are what I'd like to see added to cobweb.  If your really interested in any of these features give me a ping, it will give me a good excuse to get them done!
-
-  * remove resque requirement - I'd like the crawler to work independant of resque as a normal standalone crawler
-  * fix crawl finished notification
-  * improve internal link control
+  Feel free to contribute small or large bits of code, just please make sure that there are rspec test for the features your submitting.  We also test on travis at http://travis-ci.org/#!/stewartmckee/cobweb if you want to see the state of the project.
 
 h2. License
 

data/lib/cobweb.rb

@@ -19,7 +19,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.32"
+    "0.0.33"
   end
   
   def method_missing(method_sym, *arguments, &block)
@@ -97,7 +97,7 @@ class Cobweb
     # check if it has already been cached
     if redis.get(unique_id) and @options[:cache]
       puts "Cache hit for #{url}" unless @options[:quiet]
-      content = Marshal.load(redis.get(unique_id)).deep_symbolize_keys
+      content = deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
     else
       # this url is valid for processing so lets get on with it
       uri = Addressable::URI.parse(url.strip)
@@ -169,7 +169,7 @@ class Cobweb
             content[:body] = Base64.encode64(response.body)
           end
           content[:location] = response["location"]
-          content[:headers] = response.to_hash.deep_symbolize_keys
+          content[:headers] = deep_symbolize_keys(response.to_hash)
           # parse data for links
           link_parser = ContentLinkParser.new(content[:url], content[:body])
           content[:links] = link_parser.link_data
@@ -252,7 +252,7 @@ class Cobweb
     # check if it has already been cached
     if redis.get("head-#{unique_id}") and @options[:cache]
       puts "Cache hit for #{url}" unless @options[:quiet]
-      content = Marshal.load(redis.get("head-#{unique_id}")).deep_symbolize_keys
+      content = deep_symbolize_keys(Marshal.load(redis.get("head-#{unique_id}")))
     else
       print "Retrieving #{url }... " unless @options[:quiet]
       uri = Addressable::URI.parse(url.strip)
@@ -336,9 +336,17 @@ class Cobweb
       content
     end
   end
+  
+  def deep_symbolize_keys(hash)
+    hash.keys.each do |key|
+      value = hash[key]
+      hash.delete(key)
+      hash[key.to_sym] = value
+      if hash[key.to_sym].instance_of? Hash
+        hash[key.to_sym] = deep_symbolize_keys(hash[key.to_sym])
+      end
+    end
+    hash
+  end
+  
 end
-
-
-
-
-

data/lib/content_link_parser.rb

@@ -18,7 +18,10 @@ class ContentLinkParser
     @options[:tags][:images] = [["img[src]", "src"]]
     @options[:tags][:related] = [["link[rel]", "href"]]
     @options[:tags][:scripts] = [["script[src]", "src"]]
-    @options[:tags][:styles] = [["link[rel='stylesheet'][href]", "href"], ["style[@type^='text/css']", /url\("?(.*?)"?\)/]]    
+    @options[:tags][:styles] = [["link[rel='stylesheet'][href]", "href"], ["style[@type^='text/css']", lambda{|array,tag|
+      first_regex =/url\((['"]?)(.*?)\1\)/
+      tag.content.scan(first_regex) {|match| array << Addressable::URI.parse(match[1]).to_s}
+    }]]
     
     #clear the default tags if required
     @options[:tags] = {} if @options[:ignore_default_tags]
@@ -67,6 +70,13 @@ class ContentLinkParser
         rescue
         end
       end
+    elsif attribute.instance_of? Proc
+      @doc.css(selector).each do |tag|
+        begin
+          attribute.call(array, tag)
+        rescue
+        end
+      end
     end
   end
 

data/lib/crawl_job.rb

@@ -35,19 +35,21 @@ class CrawlJob
         # set the base url if this is the first page
         set_base_url @redis, content, content_request
         
-        internal_links = all_links_from_content(content).map{|link| link.to_s}
+        if within_queue_limits?(content_request[:crawl_limit])
+          internal_links = all_links_from_content(content).map{|link| link.to_s}
         
-        # reject the link if we've crawled it or queued it
-        internal_links.reject!{|link| @redis.sismember("crawled", link)}
-        internal_links.reject!{|link| @redis.sismember("queued", link)}
+          # reject the link if we've crawled it or queued it
+          internal_links.reject!{|link| @redis.sismember("crawled", link)}
+          internal_links.reject!{|link| @redis.sismember("queued", link)}
         
-        # select the link if its internal
-        internal_links.select!{|link| internal_link?(link)}
+          # select the link if its internal
+          internal_links.select!{|link| internal_link?(link)}
 
-        internal_links.each do |link|
-          enqueue_content(content_request, link) if within_queue_limits?(content_request[:crawl_limit])
+          internal_links.each do |link|
+            enqueue_content(content_request, link) if within_queue_limits?(content_request[:crawl_limit])
+          end
         end
-
+        
         # enqueue to processing queue
         Resque.enqueue(const_get(content_request[:processing_queue]), content.merge({:internal_urls => internal_patterns, :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]}))
         puts "#{content_request[:url]} has been sent for processing." if content_request[:debug]
@@ -74,11 +76,10 @@ class CrawlJob
           Resque.enqueue(const_get(content_request[:crawl_finished_queue]), stats.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))            
 
         end
-        
-      else
-        puts "Crawl Limit Exceeded by #{@crawl_counter - content_request[:crawl_limit].to_i} objects" if content_request[:debug]
       end
     else
+      @redis.srem "queued", content_request[:url]
+      decrement_queue_counter
       puts "Already crawled #{content_request[:url]}" if content_request[:debug]
     end
 
@@ -87,11 +88,11 @@ class CrawlJob
   private
   
   def self.within_crawl_limits?(crawl_limit)
-    crawl_limit.nil? or @crawl_counter <= crawl_limit.to_i
+    crawl_limit.nil? or @crawl_counter < crawl_limit.to_i
   end
   
   def self.within_queue_limits?(crawl_limit)
-    within_crawl_limits?(crawl_limit) and (crawl_limit.nil? or @queue_counter <= crawl_limit.to_i)
+    within_crawl_limits?(crawl_limit) and (crawl_limit.nil? or (@queue_counter + @crawl_counter) < crawl_limit.to_i)
   end
   
   def self.set_base_url(redis, content, content_request)

data/spec/cobweb/cobweb_crawler_spec.rb

@@ -17,41 +17,40 @@ describe CobwebCrawler do
 
   end  
 
-  describe "with mock" do
-
-    
-    it "should generate a cobweb_crawler object" do
-      CobwebCrawler.new.should be_an_instance_of CobwebCrawler
+  
+  it "should generate a cobweb_crawler object" do
+    CobwebCrawler.new.should be_an_instance_of CobwebCrawler
+  end
+  
+  describe "crawl" do
+    it "should crawl a site" do
+      
+      # temporary tests to run crawler - proper specs to follow.. honest
+      
+      crawler = CobwebCrawler.new({:cache => false, :quiet => false, :debug => false})
+      
+      statistics = crawler.crawl("http://rockwellcottage.heroku.com/")
+      
+      statistics.should_not be_nil
+      statistics.should be_an_instance_of Hash
+      
     end
     
-    describe "crawl" do
-      it "should crawl a site" do
-        
-        # temporary tests to run crawler - proper specs to follow.. honest
-        
-        crawler = CobwebCrawler.new({:cache => false, :quiet => false, :debug => false})
-        
-        statistics = crawler.crawl("http://rockwellcottage.heroku.com/")
-        
-        ap statistics
-        
-      end
-      
-      it "should take a block" do
+    it "should take a block" do
 
-        # temporary tests to run crawler - proper specs to follow.. honest
+      # temporary tests to run crawler - proper specs to follow.. honest
 
-        crawler = CobwebCrawler.new({:cache => false, :quiet => false, :debug => false})
-        
-        statistics = crawler.crawl("http://www.rockwellcottage.com/") do |content, statistics|
-          ap content[:url]
-          ap statistics[:average_length]
-        end
-        
-        ap statistics
-        
+      crawler = CobwebCrawler.new({:cache => false, :quiet => false, :debug => false})
+      
+      statistics = crawler.crawl("http://www.rockwellcottage.com/") do |content, statistics|
+        content[:url].should_not be_nil
+        statistics[:average_length].should_not be_nil
       end
-    end  
+      
+      statistics.should_not be_nil
+      statistics.should be_an_instance_of Hash
+      
+    end
   end  
 
 end 

data/spec/cobweb/cobweb_spec.rb

@@ -3,253 +3,185 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 describe Cobweb do
 
   before(:each) do
-  
     @base_url = "http://www.baseurl.com/"
-  
-    @default_headers = {"Cache-Control" => "private, max-age=0",
-                        "Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
-                        "Expires" => "-1",
-                        "Content-Type" => "text/html; charset=UTF-8",
-                        "Content-Encoding" => "gzip",
-                        "Transfer-Encoding" => "chunked",
-                        "Server" => "gws",
-                        "X-XSS-Protection" => "1; mode=block"}
-
     @cobweb = Cobweb.new :quiet => true, :cache => nil
-  end  
-
-  describe "with mock" do
-    before(:each) do
-      @mock_http_client = mock(Net::HTTP)
-      @mock_http_request = mock(Net::HTTPRequest)
-      @mock_http_redirect_request = mock(Net::HTTPRequest)
-      @mock_http_redirect_request2 = mock(Net::HTTPRequest)
-      
-      @mock_http_response = mock(Net::HTTPResponse)
-      @mock_http_redirect_response = mock(Net::HTTPRedirection)
-      @mock_http_redirect_response2 = mock(Net::HTTPRedirection)
-      @mock_http_get = mock(Net::HTTP::Get)
-      
-      Net::HTTP.stub!(:new).and_return(@mock_http_client)
-      Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
-      Net::HTTP::Get.stub!(:new).with("/redirect.html").and_return(@mock_http_redirect_request)
-      Net::HTTP::Get.stub!(:new).with("/redirect2.html").and_return(@mock_http_redirect_request2)
-      
-      @mock_http_client.stub!(:request).with(@mock_http_request).and_return(@mock_http_response)
-      @mock_http_client.stub!(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
-      @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)
-      @mock_http_client.stub!(:read_timeout=).and_return(nil)      
-      @mock_http_client.stub!(:open_timeout=).and_return(nil)      
-      @mock_http_client.stub!(:start).and_return(@mock_http_response)
-      @mock_http_client.stub!(:address).and_return("www.baseurl.com")
-      @mock_http_client.stub!(:port).and_return("80 ")
-      
-      @mock_http_response.stub!(:code).and_return(200)
-      @mock_http_response.stub!(:content_type).and_return("text/html")
-      @mock_http_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-      @mock_http_response.stub!(:[]).with("location").and_return(@default_headers["location"])
-      @mock_http_response.stub!(:content_length).and_return(1024)
-      @mock_http_response.stub!(:body).and_return("asdf")
-      @mock_http_response.stub!(:to_hash).and_return(@default_headers)
-      
-      @mock_http_redirect_response.stub!(:code).and_return(301)
-      @mock_http_redirect_response.stub!(:content_type).and_return("text/html")
-      @mock_http_redirect_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-      @mock_http_redirect_response.stub!(:[]).with("location").and_return("http://redirected-to.com/redirect2.html")
-      @mock_http_redirect_response.stub!(:content_length).and_return(2048)
-      @mock_http_redirect_response.stub!(:body).and_return("redirected body")
-      @mock_http_redirect_response.stub!(:to_hash).and_return(@default_headers)
-      
-      @mock_http_redirect_response2.stub!(:code).and_return(301)
-      @mock_http_redirect_response2.stub!(:content_type).and_return("text/html")
-      @mock_http_redirect_response2.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-      @mock_http_redirect_response2.stub!(:[]).with("location").and_return("http://redirected-to.com/redirected.html")
-      @mock_http_redirect_response2.stub!(:content_length).and_return(2048)
-      @mock_http_redirect_response2.stub!(:body).and_return("redirected body")
-      @mock_http_redirect_response2.stub!(:to_hash).and_return(@default_headers)
-      
+  end
+  
+  it "should generate a cobweb object" do
+    Cobweb.new.should be_an_instance_of Cobweb
+  end
+  
+  it "should setup with defaults" do
+    cobweb = Cobweb.new
+    
+    options = cobweb.instance_eval("@options")
+    
+    options[:follow_redirects].should == true
+    options[:redirect_limit].should == 10
+    options[:processing_queue].should == CobwebProcessJob
+    options[:crawl_finished_queue].should == CobwebFinishedJob
+    options[:quiet].should == true
+    options[:debug].should == false
+    options[:cache].should == 300
+    options[:timeout].should == 10
+    options[:redis_options].should == {}
+    options[:internal_urls].should == []
+    
+  end
+  
+  describe "get" do
+    it "should return a hash with default values" do
+      @cobweb.get(@base_url).should be_an_instance_of Hash
     end
     
-    it "should generate a cobweb object" do
-      Cobweb.new.should be_an_instance_of Cobweb
+    it "should return a hash with default values without quiet option" do
+      @cobweb.get(@base_url).should be_an_instance_of Hash
     end
     
-    it "should setup with defaults" do
-      cobweb = Cobweb.new
-      
-      options = cobweb.instance_eval("@options")
-      ap options
-      
-      options[:follow_redirects].should == true
-      options[:redirect_limit].should == 10
-      options[:processing_queue].should == CobwebProcessJob
-      options[:crawl_finished_queue].should == CobwebFinishedJob
-      options[:quiet].should == true
-      options[:debug].should == false
-      options[:cache].should == 300
-      options[:timeout].should == 10
-      options[:redis_options].should == {}
-      options[:internal_urls].should == []
-      
+    it "should raise exception if there is no url" do
+      lambda {@cobweb.get(nil)}.should raise_error("url cannot be nil")
     end
     
-    describe "get" do
-      it "should return a hash with default values" do
-        @cobweb.get(@base_url).should be_an_instance_of Hash
+    describe "content object" do
+      it "should return the url" do
+        @cobweb.get(@base_url)[:url].should == @base_url
+      end
+      it "should return correct content-type" do
+        @mock_http_response.stub!(:content_type).and_return("image/jpeg")
+        @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
+      end
+      it "should return correct status-code" do
+        @mock_http_response.stub!(:code).and_return(404)
+        @cobweb.get(@base_url)[:status_code].should == 404
+      end
+      it "should return correct status-code" do
+        @mock_http_response.stub!(:code).and_return(404)
+        @cobweb.get(@base_url)[:status_code].should == 404
+      end
+      it "should return correct character_set" do
+        @cobweb.get(@base_url)[:character_set].should == "UTF-8"
+      end 
+      it "should return correct content_length" do
+        @cobweb.get(@base_url)[:length].should == 1024
+      end
+      it "should return correct content_body" do
+        @cobweb.get(@base_url)[:body].should == "asdf"
+      end
+      it "should return correct location" do
+        @cobweb.get(@base_url)[:location].should == nil
+
+        @mock_http_response.stub!(:[]).with("location").and_return("http://google.com/")
+        @cobweb.get(@base_url)[:location].should == "http://google.com/"
+      end
+      it "should return correct headers" do
+        @cobweb.get(@base_url)[:headers].should == @default_headers
+      end
+      it "should return correct a hash of links" do
+        @cobweb.get(@base_url)[:links].should be_an_instance_of Hash
+      end 
+      it "should return the response time for the url" do
+        @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
+      end
+    end
+    describe "with redirect" do
+      
+      before(:each) do
+        @base_url = "http://redirect-me.com/redirect.html"
+        @cobweb = Cobweb.new(:follow_redirects => true, :quiet => true, :cache => nil)
+        
+        @mock_http_response.stub!(:[]).with("location").and_return("http://google.com/")
+        @mock_http_redirect_response.stub!(:[]).with("location").and_return("http://redirected-to.com/redirect2.html")
+        @mock_http_redirect_response2.stub!(:[]).with("location").and_return("http://redirected-to.com/redirected.html")
+        
       end
       
-      it "should return a hash with default values without quiet option" do
-        @cobweb.get(@base_url).should be_an_instance_of Hash
+      it "should flow through redirect" #do
+        
+        #@mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
+        #@mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
+        #
+        #content = @cobweb.get(@base_url)
+        #content.should be_an_instance_of HashHelper
+        #ap content
+        #content[:url].should == "http://redirect-me.com/redirect.html"
+        #content[:redirect_through].length.should == 2
+        #content[:mime_type].should == "text/html"
+        #content[:body].should == "asdf"
+        
+      #end
+      it "should return the path followed" #do
+        #@mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
+        #
+        #content = @cobweb.get(@base_url)
+        #content[:redirect_through].should == ["http://redirected-to.com/redirect2.html", "http://redirected-to.com/redirected.html"]
+        
+      #end
+      it "should not follow with redirect disabled" do
+        @cobweb = Cobweb.new(:follow_redirects => false, :cache => nil)
+        @mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
+        
+        content = @cobweb.get(@base_url)
+        content[:url].should == "http://redirect-me.com/redirect.html"
+        content[:redirect_through].should be_nil
+        content[:status_code].should == 301
+        content[:mime_type].should == "text/html"
+        content[:body].should == "redirected body"
+
       end
+    end
+    
+    describe "with cache" do
       
-      it "should raise exception if there is no url" do
-        lambda {@cobweb.get(nil)}.should raise_error("url cannot be nil")
+      before(:each) do
+        @cobweb = Cobweb.new :quiet => true, :cache => 200
       end
       
       describe "content object" do
         it "should return the url" do
           @cobweb.get(@base_url)[:url].should == @base_url
+          @cobweb.get(@base_url)[:url].should == @base_url
         end
         it "should return correct content-type" do
           @mock_http_response.stub!(:content_type).and_return("image/jpeg")
           @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
+          @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
         end
         it "should return correct status-code" do
           @mock_http_response.stub!(:code).and_return(404)
           @cobweb.get(@base_url)[:status_code].should == 404
+          @cobweb.get(@base_url)[:status_code].should == 404
         end
         it "should return correct status-code" do
           @mock_http_response.stub!(:code).and_return(404)
           @cobweb.get(@base_url)[:status_code].should == 404
+          @cobweb.get(@base_url)[:status_code].should == 404
         end
         it "should return correct character_set" do
           @cobweb.get(@base_url)[:character_set].should == "UTF-8"
+          @cobweb.get(@base_url)[:character_set].should == "UTF-8"
         end 
         it "should return correct content_length" do
           @cobweb.get(@base_url)[:length].should == 1024
+          @cobweb.get(@base_url)[:length].should == 1024
         end
         it "should return correct content_body" do
           @cobweb.get(@base_url)[:body].should == "asdf"
-        end
-        it "should return correct location" do
-          @cobweb.get(@base_url)[:location].should == nil
-  
-          @mock_http_response.stub!(:[]).with("location").and_return("http://google.com/")
-          @cobweb.get(@base_url)[:location].should == "http://google.com/"
+          @cobweb.get(@base_url)[:body].should == "asdf"
         end
         it "should return correct headers" do
           @cobweb.get(@base_url)[:headers].should == @default_headers
+          @cobweb.get(@base_url)[:headers].should == @default_headers
         end
         it "should return correct a hash of links" do
           @cobweb.get(@base_url)[:links].should be_an_instance_of Hash
+          @cobweb.get(@base_url)[:links].should be_an_instance_of Hash
         end 
         it "should return the response time for the url" do
           @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
-        end
-      end
-      describe "with redirect" do
-        
-        before(:each) do
-          @base_url = "http://redirect-me.com/redirect.html"
-          @cobweb = Cobweb.new(:follow_redirects => true, :quiet => true, :cache => nil)
-        end
-        
-        it "should flow through redirect" #do
-          
-          #@mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
-          #@mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
-          #
-          #content = @cobweb.get(@base_url)
-          #content.should be_an_instance_of HashHelper
-          #ap content
-          #content[:url].should == "http://redirect-me.com/redirect.html"
-          #content[:redirect_through].length.should == 2
-          #content[:mime_type].should == "text/html"
-          #content[:body].should == "asdf"
-          
-        #end
-        it "should return the path followed" #do
-          #@mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
-          #
-          #content = @cobweb.get(@base_url)
-          #content[:redirect_through].should == ["http://redirected-to.com/redirect2.html", "http://redirected-to.com/redirected.html"]
-          
-        #end
-        it "should not follow with redirect disabled" do
-          @cobweb = Cobweb.new(:follow_redirects => false, :cache => nil)
-          @mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
-          
-          content = @cobweb.get(@base_url)
-          content[:url].should == "http://redirect-me.com/redirect.html"
-          content[:redirect_through].should be_nil
-          content[:status_code].should == 301
-          content[:mime_type].should == "text/html"
-          content[:body].should == "redirected body"
-
+          @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
         end
       end
       
-      describe "with cache" do
-        
-        before(:each) do
-          @cobweb = Cobweb.new :quiet => true, :cache => 200
-        end
-        
-        describe "content object" do
-          it "should return the url" do
-            @cobweb.get(@base_url)[:url].should == @base_url
-            @cobweb.get(@base_url)[:url].should == @base_url
-          end
-          it "should return correct content-type" do
-            @mock_http_response.stub!(:content_type).and_return("image/jpeg")
-            @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
-            @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
-          end
-          it "should return correct status-code" do
-            @mock_http_response.stub!(:code).and_return(404)
-            @cobweb.get(@base_url)[:status_code].should == 404
-            @cobweb.get(@base_url)[:status_code].should == 404
-          end
-          it "should return correct status-code" do
-            @mock_http_response.stub!(:code).and_return(404)
-            @cobweb.get(@base_url)[:status_code].should == 404
-            @cobweb.get(@base_url)[:status_code].should == 404
-          end
-          it "should return correct character_set" do
-            @cobweb.get(@base_url)[:character_set].should == "UTF-8"
-            @cobweb.get(@base_url)[:character_set].should == "UTF-8"
-          end 
-          it "should return correct content_length" do
-            @cobweb.get(@base_url)[:length].should == 1024
-            @cobweb.get(@base_url)[:length].should == 1024
-          end
-          it "should return correct content_body" do
-            @cobweb.get(@base_url)[:body].should == "asdf"
-            @cobweb.get(@base_url)[:body].should == "asdf"
-          end
-          it "should return correct location" do
-            @cobweb.get(@base_url)[:location].should == nil
-            @cobweb.get(@base_url)[:location].should == nil
-
-            @mock_http_response.stub!(:[]).with("location").and_return("http://google.com/")
-            @cobweb.get(@base_url)[:location].should == "http://google.com/"
-            @cobweb.get(@base_url)[:location].should == "http://google.com/"
-          end
-          it "should return correct headers" do
-            @cobweb.get(@base_url)[:headers].should == @default_headers
-            @cobweb.get(@base_url)[:headers].should == @default_headers
-          end
-          it "should return correct a hash of links" do
-            @cobweb.get(@base_url)[:links].should be_an_instance_of Hash
-            @cobweb.get(@base_url)[:links].should be_an_instance_of Hash
-          end 
-          it "should return the response time for the url" do
-            @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
-            @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
-          end
-        end
-        
-      end
-    end  
+    end
   end  
 end 

data/spec/cobweb/content_link_parser_spec.rb

@@ -3,102 +3,139 @@ require File.expand_path(File.dirname(__FILE__) + '/../../lib/content_link_parse
 
 describe ContentLinkParser do
 
-  before(:each) do
-    @base_url = "http://www.baseurl.com/"
-    @content = File.read(File.dirname(__FILE__) + "/../samples/sample_html_links.html")
-    @content_parser = ContentLinkParser.new("http://sample-links.com/", @content)  
-  end
-  
-  it "should load the sample document" do
-    @content.should_not be_nil
-    @content.should_not be_empty
-  end
-  
-  it "should create a content link parser" do
-    @content_parser.should_not be_nil
-    @content_parser.should be_an_instance_of ContentLinkParser
-  end
-  
-  describe "using default tags" do
-    describe "returning general links" do
-      it "should return some links from the sample data" do
-        links = @content_parser.links 
-        links.should_not be_nil
-        links.should_not be_empty
-      end
-      it "should return the correct links" do
-        links = @content_parser.links
-        links.length.should == 4
-      end
+  describe "Sample Links Document" do
+    before(:each) do
+      @base_url = "http://www.baseurl.com/"
+      @content = File.read(File.dirname(__FILE__) + "/../samples/sample_html_links.html")
+      @content_parser = ContentLinkParser.new("http://sample-links.com/", @content)
     end
-    describe "returning image links" do
-      it "should return some image links from the sample data" do
-        links = @content_parser.images 
-        links.should_not be_nil
-        links.should_not be_empty
+
+    it "should load the sample document" do
+      @content.should_not be_nil
+      @content.should_not be_empty
+    end
+
+    it "should create a content link parser" do
+      @content_parser.should_not be_nil
+      @content_parser.should be_an_instance_of ContentLinkParser
+    end
+
+    describe "using default tags" do
+      describe "returning general links" do
+        it "should return some links from the sample data" do
+          links = @content_parser.links
+          links.should_not be_nil
+          links.should_not be_empty
+        end
+        it "should return the correct links" do
+          links = @content_parser.links
+          links.length.should == 4
+        end
       end
-      it "should return the correct links" do
-        links = @content_parser.images
-        links.length.should == 1
+      describe "returning image links" do
+        it "should return some image links from the sample data" do
+          links = @content_parser.images
+          links.should_not be_nil
+          links.should_not be_empty
+        end
+        it "should return the correct links" do
+          links = @content_parser.images
+          links.length.should == 1
+        end
       end
-    end
-    describe "returning related links" do
-      it "should return some related links from the sample data" do
-        links = @content_parser.related 
-        links.should_not be_nil
-        links.should_not be_empty
+      describe "returning related links" do
+        it "should return some related links from the sample data" do
+          links = @content_parser.related
+          links.should_not be_nil
+          links.should_not be_empty
+        end
+        it "should return the correct links" do
+          links = @content_parser.related
+          links.length.should == 2
+        end
       end
-      it "should return the correct links" do
-        links = @content_parser.related
-        links.length.should == 2
+      describe "returning script links" do
+        it "should return some script links from the sample data" do
+          links = @content_parser.scripts
+          links.should_not be_nil
+          links.should_not be_empty
+        end
+        it "should return the correct links" do
+          links = @content_parser.scripts
+          links.length.should == 1
+        end
       end
-    end
-    describe "returning script links" do
-      it "should return some script links from the sample data" do
-        links = @content_parser.scripts 
-        links.should_not be_nil
-        links.should_not be_empty
+      describe "returning style links" do
+        it "should return some style links from the sample data" do
+          links = @content_parser.styles
+          links.should_not be_nil
+          links.should_not be_empty
+        end
+        it "should return the correct links" do
+          links = @content_parser.styles
+          links.length.should == 3
+        end
       end
-      it "should return the correct links" do
-        links = @content_parser.scripts
-        links.length.should == 1
+      describe "returning unknown link type" do
+        it "should return an empty array" do
+          links = @content_parser.asdfasdfsadf
+          links.should_not be_nil
+          links.should be_an_instance_of Array
+        end
       end
     end
-    describe "returning style links" do
-      it "should return some style links from the sample data" do
-        links = @content_parser.styles 
-        links.should_not be_nil
-        links.should_not be_empty
-      end
-      it "should return the correct links" do
-        links = @content_parser.styles
-        links.length.should == 3
+
+    describe "returning all link data" do
+      it "should return a hash with all link data" do
+        link_data = @content_parser.link_data
+        link_data.should_not be_nil
+        link_data.should be_an_instance_of Hash
+
+        link_data.keys.length.should == 5
+        link_data[:links].length.should == 4
       end
     end
-    describe "returning unknown link type" do
-      it "should return an empty array" do
-        links = @content_parser.asdfasdfsadf
-        links.should_not be_nil
-        links.should be_an_instance_of Array
+
+    describe "ignoring default tags" do
+      it "should not return any links" do
+        parser = ContentLinkParser.new("http://sample-links.com", @content, :ignore_default_tags => true)
+        parser.links.should be_empty
       end
     end
   end
 
-  describe "returning all link data" do
-    it "should return a hash with all link data" do
-      link_data = @content_parser.link_data
-      link_data.should_not be_nil
-      link_data.should be_an_instance_of Hash
-      
-      link_data.keys.length.should == 5
-      link_data[:links].length.should == 4
+  describe "simple style based documents" do
+    def create_content(url)
+<<-eos
+<html>
+<head>
+<STYLE TYPE="text/css" MEDIA="screen, projection">
+  @import url(#{url});
+</STYLE>
+</head>
+</html>
+eos
     end
-  end
-  
-  describe "ignoring default tags" do
-    it "should not return any links" do
-      parser = ContentLinkParser.new("http://sample-links.com", @content, :ignore_default_tags => true)
-      parser.links.should be_empty
+    before :each do
+      @base_url = "http://www.baseurl.com/path"
+    end
+    it "should have the right link for a single quoted style" do
+      @content_parser = ContentLinkParser.new("http://sample-links.com/", create_content("'/new'"))
+      styles = @content_parser.styles
+      styles.length.should==1
+      styles[0].should=="/new"
+    end
+    it "should have the right link for a double quoted style" do
+      @content_parser = ContentLinkParser.new("http://sample-links.com/", create_content('"/new"'))
+      styles = @content_parser.styles
+      styles.length.should==1
+      styles[0].should=="/new"
+    end
+    it "should just leave links with differing quotes alone" do
+      @content_parser = ContentLinkParser.new("http://sample-links.com/", create_content('"new\''))
+      styles = @content_parser.styles
+      styles.length.should==1
+      styles[0].should=="\"new'"
     end
   end
 end 

data/spec/spec_helper.rb

@@ -8,6 +8,67 @@ RSpec.configure do |config|
     #redis_mock.stub(:new).and_return(MockRedis.new)
     
     Redis.new.flushdb
+    
+    
+    @default_headers = {"Cache-Control" => "private, max-age=0",
+                        "Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
+                        "Expires" => "-1",
+                        "Content-Type" => "text/html; charset=UTF-8",
+                        "Content-Encoding" => "",
+                        "Transfer-Encoding" => "chunked",
+                        "Server" => "gws",
+                        "X-XSS-Protection" => "1; mode=block"}
+
+    @mock_http_client = mock(Net::HTTP)
+    @mock_http_request = mock(Net::HTTPRequest)
+    @mock_http_redirect_request = mock(Net::HTTPRequest)
+    @mock_http_redirect_request2 = mock(Net::HTTPRequest)
+    
+    @mock_http_response = mock(Net::HTTPResponse)
+    @mock_http_redirect_response = mock(Net::HTTPRedirection)
+    @mock_http_redirect_response2 = mock(Net::HTTPRedirection)
+    @mock_http_get = mock(Net::HTTP::Get)
+    
+    Net::HTTP.stub!(:new).and_return(@mock_http_client)
+    Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
+    Net::HTTP::Get.stub!(:new).with("/redirect.html").and_return(@mock_http_redirect_request)
+    Net::HTTP::Get.stub!(:new).with("/redirect2.html").and_return(@mock_http_redirect_request2)
+    
+    @mock_http_client.stub!(:request).with(@mock_http_request).and_return(@mock_http_response)
+    @mock_http_client.stub!(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
+    @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)
+    @mock_http_client.stub!(:read_timeout=).and_return(nil)      
+    @mock_http_client.stub!(:open_timeout=).and_return(nil)      
+    @mock_http_client.stub!(:start).and_return(@mock_http_response)
+    @mock_http_client.stub!(:address).and_return("www.baseurl.com")
+    @mock_http_client.stub!(:port).and_return("80 ")
+    
+    @mock_http_response.stub!(:code).and_return(200)
+    @mock_http_response.stub!(:content_type).and_return("text/html")
+    @mock_http_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_response.stub!(:[]).with("location").and_return(@default_headers["location"])
+    @mock_http_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_response.stub!(:content_length).and_return(1024)
+    @mock_http_response.stub!(:body).and_return("asdf")
+    @mock_http_response.stub!(:to_hash).and_return(@default_headers)
+    
+    @mock_http_redirect_response.stub!(:code).and_return(301)
+    @mock_http_redirect_response.stub!(:content_type).and_return("text/html")
+    @mock_http_redirect_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_redirect_response.stub!(:[]).with("location").and_return("http://redirected-to.com/redirect2.html")
+    @mock_http_redirect_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_redirect_response.stub!(:content_length).and_return(2048)
+    @mock_http_redirect_response.stub!(:body).and_return("redirected body")
+    @mock_http_redirect_response.stub!(:to_hash).and_return(@default_headers)
+    
+    @mock_http_redirect_response2.stub!(:code).and_return(301)
+    @mock_http_redirect_response2.stub!(:content_type).and_return("text/html")
+    @mock_http_redirect_response2.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_redirect_response2.stub!(:[]).with("location").and_return("http://redirected-to.com/redirected.html")
+    @mock_http_redirect_response2.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_redirect_response2.stub!(:content_length).and_return(2048)
+    @mock_http_redirect_response2.stub!(:body).and_return("redirected body")
+    @mock_http_redirect_response2.stub!(:to_hash).and_return(@default_headers)
   }
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.32
+  version: 0.0.33
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-19 00:00:00.000000000 Z
+date: 2012-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70185084752540 !ruby/object:Gem::Requirement
+  requirement: &70153227362400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084752540
+  version_requirements: *70153227362400
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70185084752120 !ruby/object:Gem::Requirement
+  requirement: &70153227361980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084752120
+  version_requirements: *70153227361980
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70185084751700 !ruby/object:Gem::Requirement
+  requirement: &70153227361560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084751700
+  version_requirements: *70153227361560
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70185084751280 !ruby/object:Gem::Requirement
+  requirement: &70153227361140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084751280
+  version_requirements: *70153227361140
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70185084750860 !ruby/object:Gem::Requirement
+  requirement: &70153227360720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084750860
+  version_requirements: *70153227360720
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70185084750440 !ruby/object:Gem::Requirement
+  requirement: &70153227360300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084750440
+  version_requirements: *70153227360300
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70185084750020 !ruby/object:Gem::Requirement
+  requirement: &70153227359880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084750020
+  version_requirements: *70153227359880
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70185084749600 !ruby/object:Gem::Requirement
+  requirement: &70153227359460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084749600
+  version_requirements: *70153227359460
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70185084765540 !ruby/object:Gem::Requirement
+  requirement: &70153227359040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70185084765540
+  version_requirements: *70153227359040
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com