cobweb 0.0.42 → 0.0.43

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.42
+h1. Cobweb v0.0.43
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -20,7 +20,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.42"
+    "0.0.43"
   end
   
   def method_missing(method_sym, *arguments, &block)

data/lib/cobweb_crawler.rb

@@ -42,7 +42,7 @@ class CobwebCrawler
     puts "http://localhost:4567/statistics/#{@crawl_id}"
     puts ""
     
-    @redis.sadd "queued", base_url
+    @redis.sadd("queued", base_url) unless @redis.sismember("crawled", base_url) || @redis.sismember("queued", base_url)
     crawl_counter = @redis.scard("crawled").to_i
     queue_counter = @redis.scard("queued").to_i
 
@@ -52,8 +52,6 @@ class CobwebCrawler
         thread = Thread.new do
         
           url = @redis.spop "queued"
-          crawl_counter = @redis.scard("crawled").to_i
-          queue_counter = @redis.scard("queued").to_i
         
           @options[:url] = url
           unless @redis.sismember("crawled", url.to_s)
@@ -79,10 +77,10 @@ class CobwebCrawler
                 puts "Added #{link.to_s} to queue" if @debug
                 @redis.sadd "queued", link
               end
-            
-              crawl_counter = @redis.scard("crawled").to_i
-              queue_counter = @redis.scard("queued").to_i
-
+              
+              crawl_counter = crawl_counter + 1 #@redis.scard("crawled").to_i
+              queue_counter = queue_counter - 1 #@redis.scard("queued").to_i
+              
               @stats.update_statistics(content)
               @stats.update_status("Completed #{url}.")
               puts "Crawled: #{crawl_counter.to_i} Limit: #{@options[:crawl_limit].to_i} Queued: #{queue_counter.to_i}" if @debug 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.42
+  version: 0.0.43
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-02 00:00:00.000000000 Z
+date: 2012-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70246513464500 !ruby/object:Gem::Requirement
+  requirement: &70139604510540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513464500
+  version_requirements: *70139604510540
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70246513464080 !ruby/object:Gem::Requirement
+  requirement: &70139604510120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513464080
+  version_requirements: *70139604510120
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70246513463660 !ruby/object:Gem::Requirement
+  requirement: &70139604509700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513463660
+  version_requirements: *70139604509700
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70246513463240 !ruby/object:Gem::Requirement
+  requirement: &70139604509280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513463240
+  version_requirements: *70139604509280
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70246513462820 !ruby/object:Gem::Requirement
+  requirement: &70139604508860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513462820
+  version_requirements: *70139604508860
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70246513462400 !ruby/object:Gem::Requirement
+  requirement: &70139604508440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513462400
+  version_requirements: *70139604508440
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70246513461980 !ruby/object:Gem::Requirement
+  requirement: &70139604508020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513461980
+  version_requirements: *70139604508020
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70246513461560 !ruby/object:Gem::Requirement
+  requirement: &70139604523960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513461560
+  version_requirements: *70139604523960
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70246513461140 !ruby/object:Gem::Requirement
+  requirement: &70139604523540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513461140
+  version_requirements: *70139604523540
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70246513460720 !ruby/object:Gem::Requirement
+  requirement: &70139604523120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246513460720
+  version_requirements: *70139604523120
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com