cobweb 0.0.24 → 0.0.25

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.23
+h1. Cobweb v0.0.25
 
 h2. Intro 
   
@@ -61,9 +61,9 @@ Creates a new crawler object based on a base_url
     ** :quiet                         - hides default output (Default: false)
     ** :cache                         - sets the ttl for caching pages, set to nil to disable caching (Default: 300)
     ** :timeout                       - http timeout for requests (Default: 10)
-    ** :redis_options                 - hash containing the initialization options for redis (e.g. {:host => "redis.mydomain.com"}
-    ** :internal_urls                 - array of strings representing internal url forms for your site (eg: ['http://test.com/*', 'http://blog.test.com/*', 'http://externaltest.com/*'])
-    ** :first_page_redirect_internal  - if true and the first page crawled is a redirect, it will add the final destination of redirects to the internal_urls (e.g. http://www.test.com gets redirected to http://test.com)
+    ** :redis_options                 - hash containing the initialization options for redis (e.g. {:host => "redis.mydomain.com"}) (Default: {})
+    ** :internal_urls                 - array of strings representing internal url forms for your site (eg: ['http://test.com/*', 'http://blog.test.com/*', 'http://externaltest.com/*']) (Default: [], although your first url's scheme, host and domain are added)
+    ** :first_page_redirect_internal  - if true and the first page crawled is a redirect, it will add the final destination of redirects to the internal_urls (e.g. http://www.test.com gets redirected to http://test.com) (Default: true)
     
 bq. crawler = CobWeb.new(:follow_redirects => false)
 

data/lib/cobweb.rb

@@ -19,7 +19,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.24"
+    "0.0.25"
   end
   
   def method_missing(method_sym, *arguments, &block)

data/lib/crawl_job.rb

@@ -20,6 +20,10 @@ class CrawlJob
     
     # check we haven't crawled this url before
     unless @redis.sismember "crawled", content_request[:url]
+      @redis.srem "queued", content_request[:url]
+      decrement_queue_counter
+      @redis.sadd "crawled", content_request[:url]
+      increment_crawl_counter
       
       # if there is no limit or we're still under it lets get the url
       if content_request[:crawl_limit].nil? or @crawl_counter <= content_request[:crawl_limit].to_i
@@ -44,16 +48,31 @@ class CrawlJob
           enqueue_content(content_request, link)        
         end
 
-        # now that we're done, lets update the queues
-        @redis.srem "queued", content_request[:url]
-        decrement_queue_counter
-        @redis.sadd "crawled", content_request[:url]
-        increment_crawl_counter
-
         # enqueue to processing queue
-        Resque.enqueue(const_get(content_request[:processing_queue]), content.merge({:source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]}))
+        Resque.enqueue(const_get(content_request[:processing_queue]), content.merge({:redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]}))
         puts "#{content_request[:url]} has been sent for processing." if content_request[:debug]
         puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter}" if content_request[:debug]
+
+
+        # if the'res nothing left queued or the crawled limit has been reached
+        if @queue_counter == 0 || @crawl_counter >= content_request[:crawl_limit].to_i
+
+          puts "queue_counter: #{@queue_counter}"
+          puts "crawl_counter: #{@crawl_counter}"
+          puts "crawl_limit: #{content_request[:crawl_limit]}"
+
+          # finished
+          puts "FINISHED"
+          stats = @redis.hgetall "statistics"
+          stats[:total_pages] = @redis.get("total_pages").to_i
+          stats[:total_assets] = @redis.get("total_assets").to_i
+          stats[:crawl_counter] = @crawl_counter
+          stats[:queue_counter] = @queue_counter
+          stats[:crawled] = @redis.smembers "crawled"
+
+          Resque.enqueue(const_get(content_request[:crawl_finished_queue]), stats.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))            
+
+        end
         
       else
         puts "Crawl Limit Exceeded by #{@crawl_counter - content_request[:crawl_limit].to_i} objects" if content_request[:debug]
@@ -62,25 +81,6 @@ class CrawlJob
       puts "Already crawled #{content_request[:url]}" if content_request[:debug]
     end
 
-    # if the'res nothing left queued or the crawled limit has been reached
-    if @queue_counter == 0 || @crawl_counter >= content_request[:crawl_limit].to_i
-     
-      puts "queue_counter: #{@queue_counter}"
-      puts "crawl_counter: #{@crawl_counter}"
-      puts "crawl_limit: #{content_request[:crawl_limit]}"
-
-      # finished
-      puts "FINISHED"
-      stats = @redis.hgetall "statistics"
-      stats[:total_pages] = @redis.get "total_pages"
-      stats[:total_assets] = @redis.get "total_assets"
-      stats[:crawl_counter] = @redis.get "crawl_counter"
-      stats[:queue_counter] = @redis.get "queue_counter"
-      stats[:crawled] = @redis.smembers "crawled"
-      
-      Resque.enqueue(const_get(content_request[:crawl_finished_queue]), stats.merge({:crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))            
-      
-    end
   end
 
   private

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.24
+  version: 0.0.25
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70268501331520 !ruby/object:Gem::Requirement
+  requirement: &70349719636940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501331520
+  version_requirements: *70349719636940
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70268501331100 !ruby/object:Gem::Requirement
+  requirement: &70349719636520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501331100
+  version_requirements: *70349719636520
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70268501330680 !ruby/object:Gem::Requirement
+  requirement: &70349719636100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501330680
+  version_requirements: *70349719636100
 - !ruby/object:Gem::Dependency
-  name: addressable
-  requirement: &70268501330240 !ruby/object:Gem::Requirement
+  name: absolutize
+  requirement: &70349719635660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501330240
+  version_requirements: *70349719635660
 - !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: &70268501329820 !ruby/object:Gem::Requirement
+  name: addressable
+  requirement: &70349719635240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501329820
+  version_requirements: *70349719635240
 - !ruby/object:Gem::Dependency
-  name: awesome_print
-  requirement: &70268501329400 !ruby/object:Gem::Requirement
+  name: rspec
+  requirement: &70349719634820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501329400
+  version_requirements: *70349719634820
 - !ruby/object:Gem::Dependency
-  name: sinatra
-  requirement: &70268501328980 !ruby/object:Gem::Requirement
+  name: awesome_print
+  requirement: &70349719634400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501328980
+  version_requirements: *70349719634400
 - !ruby/object:Gem::Dependency
-  name: thin
-  requirement: &70268501328560 !ruby/object:Gem::Requirement
+  name: sinatra
+  requirement: &70349719633980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501328560
+  version_requirements: *70349719633980
 - !ruby/object:Gem::Dependency
-  name: haml
-  requirement: &70268501328140 !ruby/object:Gem::Requirement
+  name: thin
+  requirement: &70349719633560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501328140
+  version_requirements: *70349719633560
 - !ruby/object:Gem::Dependency
-  name: hashie
-  requirement: &70268501344080 !ruby/object:Gem::Requirement
+  name: haml
+  requirement: &70349719633140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,8 +120,9 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268501344080
-description: 
+  version_requirements: *70349719633140
+description: Web Crawler that uses resque background job engine to allow you to cluster
+  your crawl.
 email: stewart@rockwellcottage.com
 executables: []
 extensions: []