cobweb 0.0.31 → 0.0.32

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.31
+h1. Cobweb v0.0.32
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -19,7 +19,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.31"
+    "0.0.32"
   end
   
   def method_missing(method_sym, *arguments, &block)

data/lib/crawl_job.rb

@@ -25,7 +25,8 @@ class CrawlJob
       increment_crawl_counter
       
       # if there is no limit or we're still under it lets get the url
-      if content_request[:crawl_limit].nil? or @crawl_counter <= content_request[:crawl_limit].to_i
+      if within_crawl_limits?(content_request[:crawl_limit])
+
         content = Cobweb.new(content_request).get(content_request[:url], content_request)
         
         ## update statistics
@@ -44,14 +45,20 @@ class CrawlJob
         internal_links.select!{|link| internal_link?(link)}
 
         internal_links.each do |link|
-          enqueue_content(content_request, link)        
+          enqueue_content(content_request, link) if within_queue_limits?(content_request[:crawl_limit])
         end
 
         # enqueue to processing queue
         Resque.enqueue(const_get(content_request[:processing_queue]), content.merge({:internal_urls => internal_patterns, :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]}))
         puts "#{content_request[:url]} has been sent for processing." if content_request[:debug]
         puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter}" if content_request[:debug]
-
+        
+        #if the enqueue counter has been requested update that
+        if content_request.has_key? :enqueue_counter_key                                                                                  
+          enqueue_redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), content_request[:enqueue_counter_namespace].to_s)
+          current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
+          enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
+        end
 
         # if the'res nothing left queued or the crawled limit has been reached
         if @queue_counter == 0 || @crawl_counter >= content_request[:crawl_limit].to_i
@@ -78,6 +85,15 @@ class CrawlJob
   end
 
   private
+  
+  def self.within_crawl_limits?(crawl_limit)
+    crawl_limit.nil? or @crawl_counter <= crawl_limit.to_i
+  end
+  
+  def self.within_queue_limits?(crawl_limit)
+    within_crawl_limits?(crawl_limit) and (crawl_limit.nil? or @queue_counter <= crawl_limit.to_i)
+  end
+  
   def self.set_base_url(redis, content, content_request)
     if redis.get("base_url").nil?
       unless content[:redirect_through].nil? || content[:redirect_through].empty? || !content_request[:first_page_redirect_internal]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.31
+  version: 0.0.32
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-15 00:00:00.000000000 Z
+date: 2012-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70209995625260 !ruby/object:Gem::Requirement
+  requirement: &70185084752540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995625260
+  version_requirements: *70185084752540
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70209995624820 !ruby/object:Gem::Requirement
+  requirement: &70185084752120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995624820
+  version_requirements: *70185084752120
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70209995624400 !ruby/object:Gem::Requirement
+  requirement: &70185084751700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995624400
+  version_requirements: *70185084751700
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70209995623980 !ruby/object:Gem::Requirement
+  requirement: &70185084751280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995623980
+  version_requirements: *70185084751280
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70209995623560 !ruby/object:Gem::Requirement
+  requirement: &70185084750860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995623560
+  version_requirements: *70185084750860
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70209995623140 !ruby/object:Gem::Requirement
+  requirement: &70185084750440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995623140
+  version_requirements: *70185084750440
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70209995622720 !ruby/object:Gem::Requirement
+  requirement: &70185084750020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995622720
+  version_requirements: *70185084750020
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70209995622300 !ruby/object:Gem::Requirement
+  requirement: &70185084749600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995622300
+  version_requirements: *70185084749600
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70209995638240 !ruby/object:Gem::Requirement
+  requirement: &70185084765540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70209995638240
+  version_requirements: *70185084765540
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com