cobweb 1.0.8 → 1.0.9

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v1.0.8
+h1. Cobweb v1.0.9
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 

data/lib/cobweb.rb

@@ -42,6 +42,7 @@ class Cobweb
     default_timeout_to                        10
     default_redis_options_to                  Hash.new
     default_internal_urls_to                  []
+    default_external_urls_to                  []
     default_first_page_redirect_internal_to   true
     default_text_mime_types_to                ["text/*", "application/xhtml+xml"]
     default_obey_robots_to                    false
@@ -49,7 +50,7 @@ class Cobweb
     default_valid_mime_types_to                ["*/*"]
     default_raise_exceptions_to               false
     default_store_refered_url_to              false
-    
+
   end
   
   # This method starts the resque based crawl and enqueues the base_url

data/lib/cobweb_crawler.rb

@@ -101,7 +101,7 @@ class CobwebCrawler
             # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
             cobweb_links = CobwebLinks.new(@options)
 
-            internal_links = internal_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s))}
+            internal_links = internal_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s) && !cobweb_links.matches_external?(link))}
 
             all_internal_links = internal_links
             

data/lib/cobweb_links.rb

@@ -35,6 +35,10 @@ class CobwebLinks
   def external?(link)
     @internal_patterns.select{|pattern| link.match(pattern)}.empty? || !@external_patterns.select{|pattern| link.match(pattern)}.empty?
   end
+
+  def matches_external?(link)
+    !@external_patterns.select{|pattern| link.match(pattern)}.empty?
+  end
   
 end
 

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "1.0.8"
+    "1.0.9"
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-12 00:00:00.000000000 Z
+date: 2013-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70128767187740 !ruby/object:Gem::Requirement
+  requirement: &70315669563580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767187740
+  version_requirements: *70315669563580
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70128767183580 !ruby/object:Gem::Requirement
+  requirement: &70315669557780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767183580
+  version_requirements: *70315669557780
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70128767182220 !ruby/object:Gem::Requirement
+  requirement: &70315669553860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767182220
+  version_requirements: *70315669553860
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70128767175380 !ruby/object:Gem::Requirement
+  requirement: &70315669550620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767175380
+  version_requirements: *70315669550620
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70128767172200 !ruby/object:Gem::Requirement
+  requirement: &70315669548760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767172200
+  version_requirements: *70315669548760
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70128767170580 !ruby/object:Gem::Requirement
+  requirement: &70315669546580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767170580
+  version_requirements: *70315669546580
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70128767167800 !ruby/object:Gem::Requirement
+  requirement: &70315669542520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767167800
+  version_requirements: *70315669542520
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70128767161460 !ruby/object:Gem::Requirement
+  requirement: &70315669538180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767161460
+  version_requirements: *70315669538180
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70128767157640 !ruby/object:Gem::Requirement
+  requirement: &70315669533960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767157640
+  version_requirements: *70315669533960
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70128767152300 !ruby/object:Gem::Requirement
+  requirement: &70315669528800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767152300
+  version_requirements: *70315669528800
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70128767148180 !ruby/object:Gem::Requirement
+  requirement: &70315669525400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,7 +131,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70128767148180
+  version_requirements: *70315669525400
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more performant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface