cobweb 0.0.64 → 0.0.65

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.64
+h1. Cobweb v0.0.65
 !https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
 
 h2. Intro 

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.64"
+    "0.0.65"
   end
 
 end

data/lib/crawl_job.rb

@@ -25,7 +25,7 @@ class CrawlJob
     @debug = content_request[:debug]
     
     decrement_queue_counter
-    
+
     # check we haven't crawled this url before
     unless @redis.sismember "crawled", content_request[:url]
       # if there is no limit or we're still under it lets get the url
@@ -60,7 +60,8 @@ class CrawlJob
             @cobweb_links = CobwebLinks.new(content_request)
             if within_queue_limits?(content_request[:crawl_limit])
               internal_links = ContentLinkParser.new(content_request[:url], content[:body], content_request).all_links(:valid_schemes => [:http, :https])
-
+              #get rid of duplicate links in the same page.
+              internal_links.uniq!
               # select the link if its internal
               internal_links.select! { |link| @cobweb_links.internal?(link) }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.64
+  version: 0.0.65
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-20 00:00:00.000000000 Z
+date: 2012-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70160920762100 !ruby/object:Gem::Requirement
+  requirement: &70202662972080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920762100
+  version_requirements: *70202662972080
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70160920761600 !ruby/object:Gem::Requirement
+  requirement: &70202662971540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920761600
+  version_requirements: *70202662971540
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70160920760740 !ruby/object:Gem::Requirement
+  requirement: &70202662970680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920760740
+  version_requirements: *70202662970680
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70160920759800 !ruby/object:Gem::Requirement
+  requirement: &70202662969640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920759800
+  version_requirements: *70202662969640
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70160920758660 !ruby/object:Gem::Requirement
+  requirement: &70202662968800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920758660
+  version_requirements: *70202662968800
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70160920773700 !ruby/object:Gem::Requirement
+  requirement: &70202662967880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920773700
+  version_requirements: *70202662967880
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70160920772960 !ruby/object:Gem::Requirement
+  requirement: &70202662966760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920772960
+  version_requirements: *70202662966760
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70160920772440 !ruby/object:Gem::Requirement
+  requirement: &70202662961760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920772440
+  version_requirements: *70202662961760
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70160920771760 !ruby/object:Gem::Requirement
+  requirement: &70202662960800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70160920771760
+  version_requirements: *70202662960800
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70160920771040 !ruby/object:Gem::Requirement
+  requirement: &70202662958560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70160920771040
+  version_requirements: *70202662958560
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface