cobweb 0.0.60 → 0.0.61

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.60
+h1. Cobweb v0.0.61
 !https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
 
 h2. Intro 

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.60"
+    "0.0.61"
   end
 
 end

data/lib/crawl_job.rb

@@ -35,8 +35,11 @@ class CrawlJob
         # if there is no limit or we're still under it lets get the url
         if within_crawl_limits?(content_request[:crawl_limit])
           begin
+            # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
             @redis.srem "queued", content_request[:url]
             @redis.sadd "crawled", content_request[:url]
+            @redis.srem "queued", content[:url]
+            @redis.sadd "crawled", content[:url]
             # increment the counter if we are not limiting by page only || we are limiting count by page and it is a page
             if content_request[:crawl_limit_by_page]
               if content[:mime_type].match("text/html")

data/spec/cobweb/cobweb_job_spec.rb

@@ -207,7 +207,6 @@ def clear_queues
   Resque.queues.each do |queue|
     Resque.remove_queue(queue)
   end
-  puts "Cleared"
   
   Resque.size("cobweb_process_job").should == 0
   Resque.size("cobweb_finished_job").should == 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.60
+  version: 0.0.61
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-12 00:00:00.000000000 Z
+date: 2012-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70111532832640 !ruby/object:Gem::Requirement
+  requirement: &70301996170680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532832640
+  version_requirements: *70301996170680
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70111532832160 !ruby/object:Gem::Requirement
+  requirement: &70301996168960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532832160
+  version_requirements: *70301996168960
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70111532831300 !ruby/object:Gem::Requirement
+  requirement: &70301996168220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532831300
+  version_requirements: *70301996168220
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70111532830260 !ruby/object:Gem::Requirement
+  requirement: &70301996167040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532830260
+  version_requirements: *70301996167040
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70111532829180 !ruby/object:Gem::Requirement
+  requirement: &70301996165600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532829180
+  version_requirements: *70301996165600
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70111532844220 !ruby/object:Gem::Requirement
+  requirement: &70301996180600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532844220
+  version_requirements: *70301996180600
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70111532843480 !ruby/object:Gem::Requirement
+  requirement: &70301996179980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532843480
+  version_requirements: *70301996179980
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70111532842880 !ruby/object:Gem::Requirement
+  requirement: &70301996179420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532842880
+  version_requirements: *70301996179420
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70111532842240 !ruby/object:Gem::Requirement
+  requirement: &70301996178900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70111532842240
+  version_requirements: *70301996178900
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70111532841500 !ruby/object:Gem::Requirement
+  requirement: &70301996178200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70111532841500
+  version_requirements: *70301996178200
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface