cobweb 0.0.44 → 0.0.45

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.44
+h1. Cobweb v0.0.45
 
 h2. Intro 
   
@@ -101,6 +101,20 @@ h3. Contributing/Testing
   
     Continuous integration testing is performed by the excellent Travis: http://travis-ci.org/#!/stewartmckee/cobweb
 
+h2. Todo
+
+  * Tidy up classes with link parsing
+  * Refactoring of code to simplify design
+  * Remove requirement of redis from standalone crawler
+  * Add redis settings to standalone crawler (ie to connect to remote redis)
+  * Add ability to start and stop crawls from web interface
+  * Allow crawler to start as web interface only (ie not run crawls at start)
+  * Fix content encoding issue requiring separate process job
+
+h3. Big changes
+
+  * Refactor into a module and refactor class names to remove cobweb and increase simplicity
+
 h2. License
 
 h3. The MIT License

data/lib/cobweb.rb

@@ -20,7 +20,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.44"
+    CobwebVersion.version
   end
   
   def method_missing(method_sym, *arguments, &block)
@@ -56,9 +56,9 @@ class Cobweb
       :url => base_url 
     }  
     
-    if @options[:internal_urls].empty?
+    if @options[:internal_urls].nil? || @options[:internal_urls].empty?
       uri = Addressable::URI.parse(base_url)
-      @options[:internal_urls] << [uri.scheme, "://", uri.host, "/*"].join
+      @options[:internal_urls] = [[uri.scheme, "://", uri.host, "/*"].join]
     end
     
     request.merge!(@options)
@@ -79,6 +79,7 @@ class Cobweb
   def get(url, options = @options)
     raise "url cannot be nil" if url.nil?
     uri = Addressable::URI.parse(url)
+    uri.normalize!
     uri.fragment=nil
     url = uri.to_s
 
@@ -104,9 +105,6 @@ class Cobweb
       puts "Cache hit for #{url}" unless @options[:quiet]
       content = deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
     else
-      # this url is valid for processing so lets get on with it
-      #TODO the @http here is different from in head.  Should it be? - in head we are using a method-scoped variable.
-
       # retrieve data
       unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." unless @options[:quiet]
@@ -122,7 +120,11 @@ class Cobweb
       @http.open_timeout = @options[:timeout].to_i
       begin
         print "Retrieving #{url }... " unless @options[:quiet]
-        request = Net::HTTP::Get.new uri.request_uri
+        request_options={}
+        if options[:cookies]
+          request_options[ 'Cookie']= options[:cookies]
+        end
+        request = Net::HTTP::Get.new uri.request_uri, request_options
 
         response = @http.request request
         
@@ -135,14 +137,11 @@ class Cobweb
           # decrement redirect limit
           redirect_limit = redirect_limit - 1
 
-          # raise exception if we're being redirected to somewhere we've been redirected to in this content request          
-          #raise RedirectError("Loop detected in redirect for - #{url}") if content[:redirect_through].include? url
-          
-          # raise exception if redirect limit has reached 0
           raise RedirectError, "Redirect Limit reached" if redirect_limit == 0
+          cookies = get_cookies(response)
 
           # get the content from redirect location
-          content = get(url, options.merge(:redirect_limit => redirect_limit))
+          content = get(url, options.merge(:redirect_limit => redirect_limit, :cookies => cookies))
           content[:url] = uri.to_s
           content[:redirect_through] = [] if content[:redirect_through].nil?
           content[:redirect_through].insert(0, url)
@@ -186,7 +185,7 @@ class Cobweb
           redis.expire unique_id, @options[:cache].to_i
         end
       rescue RedirectError => e
-        puts "ERROR: #{e.message}"
+        puts "ERROR RedirectError: #{e.message}"
         
         ## generate a blank content
         content = {}
@@ -201,7 +200,7 @@ class Cobweb
         content[:links] = {}
         
       rescue SocketError => e
-        puts "ERROR: SocketError#{e.message}"
+        puts "ERROR SocketError: #{e.message}"
         
         ## generate a blank content
         content = {}
@@ -233,10 +232,20 @@ class Cobweb
     end
     content  
   end
-  
+
+  def get_cookies(response)
+    all_cookies = response.get_fields('set-cookie')
+    cookies_array = Array.new
+    all_cookies.each { |cookie|
+      cookies_array.push(cookie.split('; ')[0])
+    }
+    cookies = cookies_array.join('; ')
+  end
+
   def head(url, options = @options)
     raise "url cannot be nil" if url.nil?    
     uri = Addressable::URI.parse(url)
+    uri.normalize!
     uri.fragment=nil
     url = uri.to_s
 
@@ -255,37 +264,47 @@ class Cobweb
       redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}")
     end
     
-    content = {}
+    content = {:base_url => url}
     
     # check if it has already been cached
     if redis.get("head-#{unique_id}") and @options[:cache]
       puts "Cache hit for #{url}" unless @options[:quiet]
       content = deep_symbolize_keys(Marshal.load(redis.get("head-#{unique_id}")))
     else
-      print "Retrieving #{url }... " unless @options[:quiet]
-
       # retrieve data
-      http = Net::HTTP.new(uri.host, uri.inferred_port)
+      unless @http && @http.address == uri.host && @http.port == uri.inferred_port
+        puts "Creating connection to #{uri.host}..." unless @options[:quiet]
+        @http = Net::HTTP.new(uri.host, uri.inferred_port)
+      end
       if uri.scheme == "https"
-        http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end 
+        @http.use_ssl = true
+        @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
       
       request_time = Time.now.to_f
-      http.read_timeout = @options[:timeout].to_i
-      http.open_timeout = @options[:timeout].to_i
-      
-      begin      
-        request = Net::HTTP::Head.new uri.request_uri
-        response = http.request request
+      @http.read_timeout = @options[:timeout].to_i
+      @http.open_timeout = @options[:timeout].to_i
+      begin
+        print "Retrieving #{url }... " unless @options[:quiet]
+        request_options={}
+        if options[:cookies]
+          request_options[ 'Cookie']= options[:cookies]
+        end
+        request = Net::HTTP::Head.new uri.request_uri, request_options
+
+        response = @http.request request
 
         if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
           puts "redirected... " unless @options[:quiet]
+
           url = UriHelper.join_no_fragment(uri, response['location'])
+
           redirect_limit = redirect_limit - 1
-          options = options.clone
-          options[:redirect_limit]=redirect_limit
-          content = head(url, options)
+
+          raise RedirectError, "Redirect Limit reached" if redirect_limit == 0
+          cookies = get_cookies(response)
+
+          content = head(url, options.merge(:redirect_limit => redirect_limit, :cookies => cookies))
           content[:url] = uri.to_s
           content[:redirect_through] = [] if content[:redirect_through].nil?
           content[:redirect_through].insert(0, url)
@@ -293,7 +312,7 @@ class Cobweb
           content[:url] = uri.to_s
           content[:status_code] = response.code.to_i
           unless response.content_type.nil?
-            content[:mime_type] = response.content_type.split(";")[0].strip 
+            content[:mime_type] = response.content_type.split(";")[0].strip
             if response["Content-Type"].include? ";"
               charset = response["Content-Type"][response["Content-Type"].index(";")+2..-1] if !response["Content-Type"].nil? and response["Content-Type"].include?(";")
               charset = charset[charset.index("=")+1..-1] if charset and charset.include?("=")
@@ -310,8 +329,23 @@ class Cobweb
             puts "Not storing in cache as cache disabled" if @options[:debug]
           end
         end
+      rescue RedirectError => e
+        puts "ERROR RedirectError: #{e.message}"
+
+        ## generate a blank content
+        content = {}
+        content[:url] = uri.to_s
+        content[:response_time] = Time.now.to_f - request_time
+        content[:status_code] = 0
+        content[:length] = 0
+        content[:body] = ""
+        content[:error] = e.message
+        content[:mime_type] = "error/dnslookup"
+        content[:headers] = {}
+        content[:links] = {}
+
       rescue SocketError => e
-        puts "ERROR: #{e.message}"
+        puts "ERROR SocketError: #{e.message}"
         
         ## generate a blank content
         content = {}
@@ -326,7 +360,7 @@ class Cobweb
         content[:links] = {}
       
       rescue Timeout::Error => e
-        puts "ERROR: #{e.message}"
+        puts "ERROR Timeout::Error: #{e.message}"
         
         ## generate a blank content
         content = {}

data/lib/cobweb_crawler.rb

@@ -62,17 +62,17 @@ class CobwebCrawler
 
               @redis.sadd "crawled", url.to_s
               @redis.incr "crawl-counter" 
-            
-              internal_links = all_links_from_content(content).map{|link| link.to_s}
+              
+              internal_links = ContentLinkParser.new(url, content[:body]).all_links(:valid_schemes => [:http, :https])
 
+              # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
+              cobweb_links = CobwebLinks.new(@options)
+              internal_links = internal_links.select{|link| cobweb_links.internal?(link)}
+              
               # reject the link if we've crawled it or queued it
               internal_links.reject!{|link| @redis.sismember("crawled", link)}
               internal_links.reject!{|link| @redis.sismember("queued", link)}
-            
-
-              # select the link if its internal
-              internal_links.select!{|link| internal_link?(link)}
-
+              
               internal_links.each do |link|
                 puts "Added #{link.to_s} to queue" if @debug
                 @redis.sadd "queued", link
@@ -85,10 +85,11 @@ class CobwebCrawler
               @stats.update_statistics(content, crawl_counter, queue_counter)
               @stats.update_status("Completed #{url}.")
               puts "Crawled: #{crawl_counter.to_i} Limit: #{@options[:crawl_limit].to_i} Queued: #{queue_counter.to_i}" if @debug 
-       
-              yield content, @statistic if block_given?
+              
+              yield content, @stats.get_statistics if block_given?
 
             rescue => e
+              raise e if ENVIRONMENT == "test"
               puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
               ap e
               ap e.backtrace
@@ -105,33 +106,6 @@ class CobwebCrawler
     @stats.get_statistics
   end
   
-  
-  def internal_link?(link)
-    puts "Checking internal link for: #{link}" if @debug
-    valid_link = true
-    internal_patterns.map{|pattern| Regexp.new("^#{pattern.gsub("*", ".*?")}")}.each do |pattern|
-      puts "Matching against #{pattern.source}" if @debug
-      if link.match(pattern)
-        puts "Matched as internal" if @debug
-        return true
-      end
-    end
-    puts "Didn't match any pattern so marked as not internal" if @debug
-    false
-  end
-  
-  def internal_patterns
-    @internal_patterns ||= @redis.smembers("internal_urls")
-  end
-
-  def all_links_from_content(content)
-    links = content[:links].keys.map{|key| content[:links][key]}.flatten
-    links.reject!{|link| link.cobweb_starts_with?("javascript:")}
-    links = links.map{|link| UriHelper.join_no_fragment(content[:url], link) }
-    links.select!{|link| link.scheme == "http" || link.scheme == "https"}
-    links.uniq
-    links
-  end
 end
 
 class String

data/lib/cobweb_links.rb

@@ -0,0 +1,48 @@
+class CobwebLinks
+  
+  # processes links supplied to it
+  def initialize(options={})
+    @options = options
+    
+    raise InternalUrlsMissingError, ":internal_urls is required" unless @options.has_key? :internal_urls
+    raise InvalidUrlsError, ":internal_urls must be an array" unless @options[:internal_urls].kind_of? Array
+    raise InvalidUrlsError, ":external_urls must be an array" unless !@options.has_key?(:external_urls) || @options[:external_urls].kind_of?(Array)
+    @options[:external_urls] = [] unless @options.has_key? :external_urls
+    @options[:debug] = false unless @options.has_key? :debug
+    
+    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{pattern.gsub(".", "\\.").gsub("*", ".*?")}")}
+    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{pattern.gsub(".", "\\.").gsub("*", ".*?")}")}
+    
+  end
+  
+  def internal?(link)
+    if @options[:debug]
+      puts "--------------------------------"
+      puts "Link: #{link}"
+      puts "Internal matches"
+      ap @internal_patterns.select{|pattern| link.match(pattern)}
+      puts "External matches"
+      ap @external_patterns.select{|pattern| link.match(pattern)}
+    end
+    !@internal_patterns.select{|pattern| link.match(pattern)}.empty? && @external_patterns.select{|pattern| link.match(pattern)}.empty?
+  end
+  
+  def external?(link)
+    if @options[:debug]
+      puts "--------------------------------"
+      puts "Link: #{link}"
+      puts "Internal matches"
+      ap @internal_patterns.select{|pattern| link.match(pattern)}
+      puts "External matches"
+      ap @external_patterns.select{|pattern| link.match(pattern)}
+    end
+    @internal_patterns.select{|pattern| link.match(pattern)}.empty? || !@external_patterns.select{|pattern| link.match(pattern)}.empty?
+  end
+  
+end
+
+class InternalUrlsMissingError < Exception
+end  
+class InvalidUrlsError < Exception
+end
+

data/lib/cobweb_version.rb

@@ -0,0 +1,6 @@
+class CobwebVersion
+  def self.version
+    "0.0.45"
+  end
+
+end

data/lib/content_link_parser.rb

@@ -37,9 +37,13 @@ class ContentLinkParser
     data
   end  
   
-  def all_links
+  def all_links(options = {})    
+    options[:valid_schemes] = [:http, :https] unless options.has_key? :valid_schemes
     data = link_data
-    data.keys.map{|key| data[key]}.flatten.uniq
+    data = data.keys.map{|key| data[key]}.flatten.uniq
+    links = data.select{|link| options[:valid_schemes].include? link.split(':')[0].to_sym}
+    links = links.map{|link| UriHelper.join_no_fragment(@url, link).to_s }
+    links
   end
   
   def method_missing(m)

data/lib/crawl_job.rb

@@ -38,16 +38,17 @@ class CrawlJob
         # set the base url if this is the first page
         set_base_url @redis, content, content_request
         
+        @cobweb_links = CobwebLinks.new(content_request)
         if within_queue_limits?(content_request[:crawl_limit])
-          internal_links = all_links_from_content(content).map{|link| link.to_s}
+          internal_links = ContentLinkParser.new(content_request[:url], content[:body]).all_links(:valid_schemes => [:http, :https])
+
+          # select the link if its internal
+          internal_links.select!{|link| @cobweb_links.internal?(link)}
         
           # reject the link if we've crawled it or queued it
           internal_links.reject!{|link| @redis.sismember("crawled", link)}
           internal_links.reject!{|link| @redis.sismember("queued", link)}
-        
-          # select the link if its internal
-          internal_links.select!{|link| internal_link?(link)}
-
+          
           internal_links.each do |link|
             enqueue_content(content_request, link) if within_queue_limits?(content_request[:crawl_limit])
           end
@@ -83,11 +84,11 @@ class CrawlJob
   def self.finished(content_request)
     # finished
     @stats.end_crawl(content_request)
-    Resque.enqueue(const_get(content_request[:crawl_finished_queue]), Stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))                
+    Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))
   end
 
   def self.send_to_processing_queue(content, content_request)
-    content_to_send = content.merge({:internal_urls => internal_patterns, :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]})
+    content_to_send = content.merge({:internal_urls => content_request[:internal_urls], :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]})
     if content_request[:use_encoding_safe_process_job]
       content_to_send[:body] = Base64.encode64(content[:body])
       content_to_send[:processing_queue] = content_request[:processing_queue]
@@ -119,33 +120,6 @@ class CrawlJob
     end
   end
   
-  def self.internal_link?(link)
-    puts "Checking internal link for: #{link}" if @debug
-    valid_link = true
-    internal_patterns.map{|pattern| Regexp.new("^#{pattern.gsub("*", ".*?")}")}.each do |pattern|
-      puts "Matching against #{pattern.source}" if @debug
-      if link.match(pattern)
-        puts "Matched as internal" if @debug
-        return true
-      end
-    end
-    puts "Didn't match any pattern so marked as not internal" if @debug
-    false
-  end
-  
-  def self.internal_patterns
-    @internal_patterns ||= @redis.smembers("internal_urls")
-  end
-
-  def self.all_links_from_content(content)
-    links = content[:links].keys.map{|key| content[:links][key]}.flatten
-    links.reject!{|link| link.starts_with?("javascript:")}
-    links = links.map{|link| UriHelper.join_no_fragment(content[:url], link) }
-    links.select!{|link| link.scheme == "http" || link.scheme == "https"}
-    links.uniq
-    links
-  end
-
   def self.enqueue_content(content_request, link)
     new_request = content_request.clone
     new_request[:url] = link

data/spec/cobweb/cobweb_links_spec.rb

@@ -0,0 +1,103 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../../lib/cobweb_links')
+
+describe CobwebLinks do
+
+  before(:each) do
+  
+    @base_url = "http://www.baseurl.com/"
+  
+    @default_headers = {"Cache-Control" => "private, max-age=0",
+                        "Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
+                        "Expires" => "-1",
+                        "Content-Type" => "text/html; charset=UTF-8",
+                        "Content-Encoding" => "gzip",
+                        "Transfer-Encoding" => "chunked",
+                        "Server" => "gws",
+                        "X-XSS-Protection" => "1; mode=block"}
+
+  end  
+
+  
+  it "should generate a cobweb_links object" do
+    CobwebLinks.new(:internal_urls => [""]).should be_an_instance_of CobwebLinks
+  end
+
+  it "should raise error with no internal links" do
+    expect {CobwebLinks.new()}.to raise_error(InternalUrlsMissingError)
+  end    
+  it "should not raise error with missing external links" do
+    expect {CobwebLinks.new(:internal_urls => ["http://domain_one.com/"])}.to_not raise_error(InternalUrlsMissingError)
+  end
+  it "should raise error with invalid internal links" do
+    expect {CobwebLinks.new(:internal_urls => "")}.to raise_error(InvalidUrlsError)
+  end
+  it "should raise error with invalid external links" do
+    expect {CobwebLinks.new(:internal_urls => [], :external_urls => "")}.to raise_error(InvalidUrlsError)
+  end
+    
+  
+  describe "internal and external links" do
+    it "should only return internal links" do
+      cobweb_links = CobwebLinks.new(:internal_urls => ["http://domain_one.com/"], :external_urls => ["http://domain_two.com/"])
+      cobweb_links.internal?("http://domain_one.com/pageone.html").should be_true
+      cobweb_links.internal?("http://domain_one.com/pagetwo.html").should be_true
+    end
+    it "should not return external links" do
+      cobweb_links = CobwebLinks.new(:internal_urls => ["http://domain_one.com/"], :external_urls => ["http://domain_two.com/"])
+      cobweb_links.external?("http://domain_one.com/pageone.html").should be_false
+      cobweb_links.external?("http://domain_two.com/pageone.html").should be_true      
+      cobweb_links.external?("http://external.com/pageone.html").should be_true      
+    end
+    it "should override internal links with external links" do
+      cobweb_links = CobwebLinks.new(:internal_urls => ["http://domain_one.com/"], :external_urls => ["http://domain_one.com/blog"])
+      cobweb_links.internal?("http://domain_one.com/pageone.html").should be_true
+      cobweb_links.external?("http://domain_one.com/pageone.html").should be_false
+      cobweb_links.internal?("http://domain_one.com/blog/pageone.html").should be_false
+      cobweb_links.external?("http://domain_one.com/blog/pageone.html").should be_true
+      cobweb_links.internal?("http://domain_two.com/blog/pageone.html").should be_false
+      cobweb_links.external?("http://domain_two.com/blog/pageone.html").should be_true
+    end
+  end
+  it "should only match from beginning of url" do
+    cobweb_links = CobwebLinks.new(:internal_urls => ["http://www.domain_one.com/"], :external_urls => ["http://www.domain_two.com/"])
+    cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_true
+    cobweb_links.internal?("http://www.domain_two.com/pageone.html").should be_false
+    cobweb_links.internal?("http://www.domain_one.com/pageone.html?url=http://www.domain_two.com/pageone.html").should be_true
+    cobweb_links.internal?("http://www.domain_two.com/pageone.html?url=http://www.domain_one.com/pageone.html").should be_false
+  end
+  
+  describe "using wildcards" do
+    it "should match internal links with wildcards" do
+      cobweb_links = CobwebLinks.new(:internal_urls => ["http://*.domain_one.com/"], :external_urls => ["http://blog.domain_one.com/"])
+      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_true
+      cobweb_links.internal?("http://images.domain_one.com/logo.png").should be_true      
+      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_false
+    end
+    it "should match external links with wildcards" do
+      cobweb_links = CobwebLinks.new(:internal_urls => ["http://www.domain_one.com/"], :external_urls => ["http://*.domain_one.com/"])
+      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_false
+      cobweb_links.internal?("http://images.domain_one.com/logo.png").should be_false
+      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_false
+    end
+    it "should allow multiple wildcards" do
+      cobweb_links = CobwebLinks.new(:internal_urls => ["http://*.*.domain_one.com/"])
+      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_false
+      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_false
+      cobweb_links.internal?("http://www.marketing.domain_one.com/pageone.html").should be_true
+      cobweb_links.internal?("http://blog.designers.domain_one.com/pagetwo.html").should be_true      
+    end
+    it "should allow multiple country tlds with wildcards" do
+      cobweb_links = CobwebLinks.new(:internal_urls => ["http://*.domain_one.*/", "http://*.domain_one.*.*/"])
+      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_true
+      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_true
+      cobweb_links.internal?("http://www.domain_one.co.uk/pageone.html").should be_true
+      cobweb_links.internal?("http://blog.domain_one.co.uk/pageone.html").should be_true
+      cobweb_links.internal?("http://www.domain_one.com.au/pageone.html").should be_true
+      cobweb_links.internal?("http://blog.domain_one.com.au/pageone.html").should be_true
+      cobweb_links.internal?("http://www.domain_one.ie/pageone.html").should be_true
+      cobweb_links.internal?("http://blog.domain_one.ie/pageone.html").should be_true
+    end
+  end
+  
+end

data/spec/cobweb/cobweb_spec.rb

@@ -186,22 +186,22 @@ describe Cobweb do
     describe "location setting" do
       it "Get should strip fragments" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/")
+        Net::HTTP::Get.should_receive(:new).with("/", {})
         @cobweb.get("http://www.google.com/#ignore")
       end
       it "head should strip fragments" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Head.should_receive(:new).with("/").and_return(@mock_http_request)
+        Net::HTTP::Head.should_receive(:new).with("/", {}).and_return(@mock_http_request)
         @cobweb.head("http://www.google.com/#ignore")
       end
       it "get should not strip path" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff")
+        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff", {})
         @cobweb.get("http://www.google.com/path/to/stuff#ignore")
       end
       it "get should not strip query string" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string")
+        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string", {})
         @cobweb.get("http://www.google.com/path/to/stuff?query_string#ignore")
       end
     end

data/spec/cobweb/content_link_parser_spec.rb

@@ -29,7 +29,7 @@ describe ContentLinkParser do
         end
         it "should return the correct links" do
           links = @content_parser.links
-          links.length.should == 4
+          links.length.should == 7
         end
       end
       describe "returning image links" do
@@ -92,7 +92,17 @@ describe ContentLinkParser do
         link_data.should be_an_instance_of Hash
 
         link_data.keys.length.should == 5
-        link_data[:links].length.should == 4
+        link_data[:links].length.should == 7
+      end
+      
+      it "should return all http and https links by default" do
+        links = @content_parser.all_links
+        links.count.should == 9
+      end
+      
+      it "should return only valid_schemes supplied" do
+        links = @content_parser.all_links(:valid_schemes => [:https])
+        links.count.should == 1
       end
     end
 

data/spec/samples/sample_html_links.html

@@ -23,7 +23,10 @@
 
 <body bgcolor="#FFFFFF"><!-- #BeginLibraryItem "/Library/navtop.lbi" --></p>
 
-<a href="http://sampleurl-a.com/">Click Here for Sample URL 1</a>
+  <a href="http://sampleurl-a.com/">Click Here for Sample URL 1</a>
+  <a href="mailto:stewart@theizone.co.uk">Click Here to email </a>
+  <a href="javascript:alert('javascript clicked');">click here for javscript</a>
+  <a href="https://sampleurl-a.com/">Click Here for SSL link to URL 1</a>
 <frameset><frame src="http://sampleurl-frame.com/"></frame></frameset>
 
 <map id="testmap"><area href="http://sampleurl-area"></area>></map>

data/spec/spec_helper.rb

@@ -1,14 +1,16 @@
 require File.expand_path(File.dirname(__FILE__) + '/../lib/cobweb')
 require 'mock_redis'
 
+ENVIRONMENT = "test"
+
 RSpec.configure do |config|
   config.before(:each) {
+        
     redis_mock = double("redis")
     redis_mock.stub(:new).and_return(@redis_mock_object)
     
     #redis_mock.flushdb
     
-    
     @default_headers = {"Cache-Control" => "private, max-age=0",
                         "Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
                         "Expires" => "-1",
@@ -39,8 +41,8 @@ RSpec.configure do |config|
     
     Net::HTTP.stub!(:new).and_return(@mock_http_client)
     Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect.html").and_return(@mock_http_redirect_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect2.html").and_return(@mock_http_redirect_request2)
+    Net::HTTP::Get.stub!(:new).with("/redirect.html", {}).and_return(@mock_http_redirect_request)
+    Net::HTTP::Get.stub!(:new).with("/redirect2.html", {}).and_return(@mock_http_redirect_request2)
 
     Net::HTTP::Head.stub!(:new).and_return(@mock_http_request)
 
@@ -60,6 +62,7 @@ RSpec.configure do |config|
     @mock_http_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
     @mock_http_response.stub!(:content_length).and_return(1024)
     @mock_http_response.stub!(:body).and_return("asdf")
+    @mock_http_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
     @mock_http_response.stub!(:to_hash).and_return(@default_headers)
     
     @mock_http_redirect_response.stub!(:code).and_return(301)
@@ -69,6 +72,7 @@ RSpec.configure do |config|
     @mock_http_redirect_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
     @mock_http_redirect_response.stub!(:content_length).and_return(2048)
     @mock_http_redirect_response.stub!(:body).and_return("redirected body")
+    @mock_http_redirect_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
     @mock_http_redirect_response.stub!(:to_hash).and_return(@default_headers)
     
     @mock_http_redirect_response2.stub!(:code).and_return(301)
@@ -78,6 +82,7 @@ RSpec.configure do |config|
     @mock_http_redirect_response2.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
     @mock_http_redirect_response2.stub!(:content_length).and_return(2048)
     @mock_http_redirect_response2.stub!(:body).and_return("redirected body")
+    @mock_http_redirect_response2.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
     @mock_http_redirect_response2.stub!(:to_hash).and_return(@default_headers)
   }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.44
+  version: 0.0.45
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-03 00:00:00.000000000 Z
+date: 2012-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70139600146360 !ruby/object:Gem::Requirement
+  requirement: &70107297058680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600146360
+  version_requirements: *70107297058680
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70139600145940 !ruby/object:Gem::Requirement
+  requirement: &70107297056880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600145940
+  version_requirements: *70107297056880
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70139600145520 !ruby/object:Gem::Requirement
+  requirement: &70107297056360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600145520
+  version_requirements: *70107297056360
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70139600145100 !ruby/object:Gem::Requirement
+  requirement: &70107297055840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600145100
+  version_requirements: *70107297055840
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70139600144680 !ruby/object:Gem::Requirement
+  requirement: &70107297055120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600144680
+  version_requirements: *70107297055120
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70139600144260 !ruby/object:Gem::Requirement
+  requirement: &70107297054400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600144260
+  version_requirements: *70107297054400
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70139600143840 !ruby/object:Gem::Requirement
+  requirement: &70107297053900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600143840
+  version_requirements: *70107297053900
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70139600143420 !ruby/object:Gem::Requirement
+  requirement: &70107297053360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600143420
+  version_requirements: *70107297053360
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70139600143000 !ruby/object:Gem::Requirement
+  requirement: &70107297052800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,18 +109,18 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70139600143000
+  version_requirements: *70107297052800
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70139600142580 !ruby/object:Gem::Requirement
+  requirement: &70107297052240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70139600142580
+  version_requirements: *70107297052240
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com
@@ -131,6 +131,7 @@ extra_rdoc_files:
 files:
 - spec/cobweb/cobweb_crawler_spec.rb
 - spec/cobweb/cobweb_job_spec.rb
+- spec/cobweb/cobweb_links_spec.rb
 - spec/cobweb/cobweb_spec.rb
 - spec/cobweb/content_link_parser_spec.rb
 - spec/samples/sample_html_links.html
@@ -139,7 +140,9 @@ files:
 - lib/cobweb.rb
 - lib/cobweb_crawler.rb
 - lib/cobweb_finished_job.rb
+- lib/cobweb_links.rb
 - lib/cobweb_process_job.rb
+- lib/cobweb_version.rb
 - lib/content_link_parser.rb
 - lib/crawl_job.rb
 - lib/encoding_safe_process_job.rb