cobweb 0.0.74 → 0.0.75

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.74
+h1. Cobweb v0.0.75
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 

data/lib/cobweb.rb

@@ -124,7 +124,7 @@ class Cobweb
     else
       # retrieve data
       unless @http && @http.address == uri.host && @http.port == uri.inferred_port
-        puts "Creating connection to #{uri.host}..." unless @options[:quiet]
+        puts "Creating connection to #{uri.host}..." if @options[:debug]
         @http = Net::HTTP.new(uri.host, uri.inferred_port)
       end
       if uri.scheme == "https"

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.74"
+    "0.0.75"
   end
 
 end

data/lib/content_link_parser.rb

@@ -13,6 +13,7 @@ class ContentLinkParser
     base_url = @url.to_s
     if @doc.at("base[href]")
       base_url = @doc.at("base[href]").attr("href").to_s
+      @url = base_url if base_url
     end
 
     @options[:tags] = {}

data/lib/crawl.rb

@@ -1,26 +1,26 @@
 module CobwebModule
   class Crawl
-    
+
     def initialize(options={})
       @options = HashUtil.deep_symbolize_keys(options)
-      
+
       setup_defaults
       @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@options[:crawl_id]}", Redis.new(@options[:redis_options]))
       @stats = Stats.new(@options)
       @debug = @options[:debug]
       @first_to_finish = false
-      
+
     end
-    
+
     # Returns true if the url requested is already in the crawled queue
     def already_crawled?(link=@options[:url])
-       @redis.sismember "crawled", link
+      @redis.sismember "crawled", link
     end
-    
+
     def already_queued?(link)
       @redis.sismember "queued", link
     end
-    
+
     # Returns true if the crawl count is within limits
     def within_crawl_limits?
       @options[:crawl_limit].nil? || crawl_counter < @options[:crawl_limit].to_i
@@ -33,51 +33,60 @@ module CobwebModule
 
     # Returns true if the queue count is calculated to be still within limits when complete
     def within_queue_limits?
-      
+
       # if we are limiting by page we can't limit the queue size as we don't know the mime type until retrieved
       if @options[:crawl_limit_by_page]
         return true
-        
-      # if a crawl limit is set, limit queue size to crawled + queue
+
+        # if a crawl limit is set, limit queue size to crawled + queue
       elsif @options[:crawl_limit].to_i > 0
         (queue_counter + crawl_counter) < @options[:crawl_limit].to_i
-      
-      # no crawl limit set so always within queue limit
+
+        # no crawl limit set so always within queue limit
       else
         true
       end
     end
-    
+
     def retrieve
-      unless already_crawled?
-        if within_crawl_limits?
-          @stats.update_status("Retrieving #{@options[:url]}...")
-          @content = Cobweb.new(@options).get(@options[:url], @options)
-          if @options[:url] == @redis.get("original_base_url")
-             @redis.set("crawled_base_url", @content[:base_url])
-          end
-          update_queues
-      
-          if content.permitted_type?
-            ## update statistics
-          
-            @stats.update_statistics(@content)
-            return true
+      lock("retrieve") do
+        unless @redis.sismember("currently_running", @options[:url])
+          @redis.sadd("currently_running", @options[:url])
+          unless already_crawled?
+            if within_crawl_limits?
+              @stats.update_status("Retrieving #{@options[:url]}...")
+              @content = Cobweb.new(@options).get(@options[:url], @options)
+              if @options[:url] == @redis.get("original_base_url")
+                @redis.set("crawled_base_url", @content[:base_url])
+              end
+              update_queues
+
+              if content.permitted_type?
+                ## update statistics
+
+                @stats.update_statistics(@content)
+                return true
+              end
+            else
+              decrement_queue_counter
+            end
+          else
+            decrement_queue_counter
           end
         else
+          debug_puts "\n\nDETECTED DUPLICATE JOB for #{@options[:url]}\n"
+          debug_ap @redis.smembers("currently_running")
           decrement_queue_counter
         end
-      else
-        decrement_queue_counter
+        false
       end
-      false
     end
-    
+
     def process_links &block
-      
+
       # set the base url if this is the first page
       set_base_url @redis
-      
+
       @cobweb_links = CobwebLinks.new(@options)
       if within_queue_limits?
         internal_links = ContentLinkParser.new(@options[:url], content.body, @options).all_links(:valid_schemes => [:http, :https])
@@ -99,20 +108,20 @@ module CobwebModule
                 increment_queue_counter
               end
             else
-              puts "Cannot enqueue new content as crawl has been cancelled." if @options[:debug]
+              debug_puts "Cannot enqueue new content as crawl has been cancelled."
             end
           end
         end
       end
     end
-    
+
     def content
       raise "Content is not available" if @content.nil?
-      CobwebModule::CrawlObject.new(@content, @options) 
+      CobwebModule::CrawlObject.new(@content, @options)
     end
-    
+
     def update_queues
-      @redis.multi do
+      lock("update_queues") do
         #@redis.incr "inprogress"
         # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
         @redis.srem "queued", @options[:url]
@@ -123,7 +132,6 @@ module CobwebModule
         end
         # increment the counter if we are not limiting by page only || we are limiting count by page and it is a page
         if @options[:crawl_limit_by_page]
-          ap "#{content.mime_type} - #{content.url}"
           if content.mime_type.match("text/html")
             increment_crawl_counter
           end
@@ -133,12 +141,12 @@ module CobwebModule
         decrement_queue_counter
       end
     end
-    
+
     def to_be_processed?
-      !finished? || first_to_finish? || within_process_limits?
+      (!finished? || within_process_limits?) && !@redis.sismember("enqueued", @options[:url])
     end
-    
-    def process
+
+    def process(&block)
       if @options[:crawl_limit_by_page]
         if content.mime_type.match("text/html")
           increment_process_counter
@@ -146,34 +154,42 @@ module CobwebModule
       else
         increment_process_counter
       end
+      @redis.sadd "enqueued", @options[:url]
+      
+      yield if block_given?
+      @redis.incr("crawl_job_enqueued_count")
     end
-    
+
+    def finished_processing
+      @redis.srem "currently_running", @options[:url]
+    end
+
     def finished?
       print_counters
-      # if there's nothing left queued or the crawled limit has been reached
+      # if there's nothing left queued or the crawled limit has been reached and we're not still processing something
       if @options[:crawl_limit].nil? || @options[:crawl_limit] == 0
-        if queue_counter.to_i == 0
+        if queue_counter == 0 && @redis.smembers("currently_running").empty?
           finished
           return true
         end
-      elsif (queue_counter.to_i) == 0 || crawl_counter.to_i >= @options[:crawl_limit].to_i
+      elsif (queue_counter == 0 && @redis.smembers("currently_running").empty?) || process_counter >= @options[:crawl_limit].to_i
         finished
         return true
       end
       false
     end
-    
+
     def finished
-      set_first_to_finish if !@redis.exists("first_to_finish")
-      ap "CRAWL FINISHED  #{@options[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if @options[:debug]
+      set_first_to_finish 
+      debug_ap "CRAWL FINISHED  #{@options[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}"
       @stats.end_crawl(@options)
     end
-    
+
     def set_first_to_finish
       @redis.watch("first_to_finish") do
         if !@redis.exists("first_to_finish")
           @redis.multi do
-            puts "set first to finish"
+            debug_puts "set first to finish"
             @first_to_finish = true
             @redis.set("first_to_finish", 1)
           end
@@ -182,23 +198,51 @@ module CobwebModule
         end
       end
     end
-    
-    
-    def first_to_finish? 
+
+
+    def first_to_finish?
       @first_to_finish
     end
     
     def crawled_base_url
       @redis.get("crawled_base_url")
     end
-    
+
     def statistics
       @stats.get_statistics
     end
-    
+
     def redis
       @redis
     end
+
+    def lock(key, &block)
+      debug_puts "REQUESTING LOCK [#{key}]"
+      set_nx = @redis.setnx("#{key}_lock", "locked")
+      debug_puts "LOCK:#{key}:#{set_nx}"
+      while !set_nx
+        debug_puts "===== WAITING FOR LOCK [#{key}] ====="
+        sleep 0.01
+        set_nx = @redis.setnx("#{key}_lock", "locked")
+      end
+
+      debug_puts "RECEIVED LOCK [#{key}]"
+      begin
+        result = yield
+      ensure
+        @redis.del("#{key}_lock")
+        debug_puts "LOCK RELEASED [#{key}]"
+      end
+      result
+    end
+    
+    def debug_ap(value)
+      ap(value) if @options[:debug]
+    end
+    
+    def debug_puts(value)
+      puts(value) if @options[:debug]
+    end
     
     private
     def setup_defaults
@@ -206,7 +250,7 @@ module CobwebModule
       @options[:crawl_limit_by_page] = false unless @options.has_key? :crawl_limit_by_page
       @options[:valid_mime_types] = ["*/*"] unless @options.has_key? :valid_mime_types
     end
-    
+
     # Increments the queue counter and refreshes crawl counters
     def increment_queue_counter
       @redis.incr "queue-counter"
@@ -223,7 +267,7 @@ module CobwebModule
     def decrement_queue_counter
       @redis.decr "queue-counter"
     end
-    
+
     def crawl_counter
       @redis.get("crawl-counter").to_i
     end
@@ -233,19 +277,19 @@ module CobwebModule
     def process_counter
       @redis.get("process-counter").to_i
     end
-    
+
     def status
       @stats.get_status
     end
-    
+
     def print_counters
-      puts counters
+      debug_puts counters
     end
-    
+
     def counters
-      "crawl_counter: #{crawl_counter} queue_counter: #{queue_counter} process_counter: #{process_counter} crawl_limit: #{@options[:crawl_limit]}"
+      "crawl_counter: #{crawl_counter} queue_counter: #{queue_counter} process_counter: #{process_counter} crawl_limit: #{@options[:crawl_limit]} currently_running: #{@redis.smembers("currently_running").count}"
     end
-    
+
     # Sets the base url in redis.  If the first page is a redirect, it sets the base_url to the destination
     def set_base_url(redis)
       if redis.get("base_url").nil?
@@ -257,7 +301,5 @@ module CobwebModule
       end
     end
 
-
-    
   end
-end
+end

data/lib/crawl_job.rb

@@ -24,40 +24,49 @@ class CrawlJob
         @crawl.process_links do |link|
 
           # enqueue the links to resque
-          puts "ENQUEUED LINK: #{link}"
+          @crawl.debug_puts "ENQUEUED LINK: #{link}" 
           enqueue_content(content_request, link) 
 
         end
         
-        
-        if @crawl.to_be_processed?
-          @crawl.process
-          
-          # enqueue to processing queue
-          @crawl.redis.incr("crawl_job_enqueued_count")
-          puts "ENQUEUED [#{@crawl.redis.get("crawl_job_enqueued_count")}] URL: #{@crawl.content.url}"
-          send_to_processing_queue(@crawl.content.to_hash, content_request)
-          
-          
-          #if the enqueue counter has been requested update that
-          if content_request.has_key?(:enqueue_counter_key)
-            enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => Redis.new(content_request[:redis_options]))
-            current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
-            enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
+        @crawl.lock("crawl_job_process") do
+          if @crawl.to_be_processed?
+            
+            @crawl.process do
+
+              # enqueue to processing queue
+              @crawl.debug_puts "ENQUEUED [#{@crawl.redis.get("crawl_job_enqueued_count")}] URL: #{@crawl.content.url}"
+              send_to_processing_queue(@crawl.content.to_hash, content_request)
+
+              #if the enqueue counter has been requested update that
+              if content_request.has_key?(:enqueue_counter_key)
+                enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => Redis.new(content_request[:redis_options]))
+                current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
+                enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
+              end
+              
+            end
+          else
+            @crawl.debug_puts "@crawl.finished? #{@crawl.finished?}"
+            @crawl.debug_puts "@crawl.within_crawl_limits? #{@crawl.within_crawl_limits?}"
+            @crawl.debug_puts "@crawl.first_to_finish? #{@crawl.first_to_finish?}"
           end
-        else
-          ap "@crawl.finished? #{@crawl.finished?}"
-          ap "@crawl.within_crawl_limits? #{@crawl.within_crawl_limits?}"
-          ap "@crawl.first_to_finish? #{@crawl.first_to_finish?}"
+          
         end
       end
     end
+    
+    @crawl.lock("finished") do
+      # let the crawl know we're finished with this object
+      @crawl.finished_processing
 
-    # test queue and crawl sizes to see if we have completed the crawl
-    ap "finished? #{@crawl.finished?}"
-    ap "first_to_finish? #{@crawl.first_to_finish?}" if @crawl.finished?
-    if @crawl.finished? && @crawl.first_to_finish?
-      finished(content_request)
+      # test queue and crawl sizes to see if we have completed the crawl
+      @crawl.debug_puts "finished? #{@crawl.finished?}"
+      @crawl.debug_puts "first_to_finish? #{@crawl.first_to_finish?}" if @crawl.finished?
+      if @crawl.finished? && @crawl.first_to_finish?
+        @crawl.debug_puts "Calling crawl_job finished"
+        finished(content_request)
+      end
     end
     
   end
@@ -68,6 +77,7 @@ class CrawlJob
     additional_stats[:redis_options] = content_request[:redis_options] unless content_request[:redis_options] == {}
     additional_stats[:source_id] = content_request[:source_id] unless content_request[:source_id].nil?
     
+    @crawl.debug_puts "increment crawl_finished_enqueued_count"
     @crawl.redis.incr("crawl_finished_enqueued_count")
     Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @crawl.statistics.merge(additional_stats))
   end
@@ -85,7 +95,7 @@ class CrawlJob
     else
       Resque.enqueue(const_get(content_request[:processing_queue]), content_to_send)
     end
-    puts "#{content_request[:url]} has been sent for processing. use_encoding_safe_process_job: #{content_request[:use_encoding_safe_process_job]}" if content_request[:debug]
+    @crawl.debug_puts "#{content_request[:url]} has been sent for processing. use_encoding_safe_process_job: #{content_request[:use_encoding_safe_process_job]}"
   end
 
   private

data/spec/cobweb/cobweb_job_spec.rb

@@ -9,7 +9,7 @@ describe Cobweb, :local_only => true do
     # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES
     puts "Starting Workers... Please Wait..."
     `mkdir log`
-    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=3 QUEUE=cobweb_crawl_job > log/output.log &")
+    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=10 QUEUE=cobweb_crawl_job > log/output.log &")
     puts "Workers Started."
 
   end
@@ -42,16 +42,16 @@ describe Cobweb, :local_only => true do
       @redis.get("crawl_job_enqueued_count").to_i.should == 0
     end
 
-    it "should not complete the crawl when cancelled" do
-      crawl = @cobweb.start(@base_url)
-      crawl_obj = CobwebCrawlHelper.new(crawl)
-      sleep 6
-      crawl_obj.destroy
-      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
-      wait_for_crawl_finished crawl[:crawl_id]
-      @redis.get("crawl_job_enqueued_count").to_i.should > 0
-      @redis.get("crawl_job_enqueued_count").to_i.should_not == @base_page_count
-    end
+    # it "should not complete the crawl when cancelled" do
+    #   crawl = @cobweb.start(@base_url)
+    #   crawl_obj = CobwebCrawlHelper.new(crawl)
+    #   sleep 6
+    #   crawl_obj.destroy
+    #   @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+    #   wait_for_crawl_finished crawl[:crawl_id]
+    #   @redis.get("crawl_job_enqueued_count").to_i.should > 0
+    #   @redis.get("crawl_job_enqueued_count").to_i.should_not == @base_page_count
+    # end
 
   end
   describe "with no crawl limit" do
@@ -83,11 +83,13 @@ describe Cobweb, :local_only => true do
       @redis.get("crawl_finished_enqueued_count").to_i.should == 1
     end
   end
+
   describe "with limited mime_types" do
     before(:each) do
       @request = {
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
-        :quiet => true,
+        :quiet => false,
+        :debug => false,
         :cache => nil,
         :valid_mime_types => ["text/html"]
       }
@@ -112,12 +114,26 @@ describe Cobweb, :local_only => true do
     before(:each) do
       @request = {
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
-        :quiet => true,
+        :quiet => false,
+        :debug => false,
         :cache => nil
       }
       @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", Redis.new)
     end
 
+    # describe "crawling http://yepadeperrors.wordpress.com/ with limit of 20" do
+    #   before(:each) do
+    #     @request[:crawl_limit] = 20
+    #     @cobweb = Cobweb.new @request
+    #   end
+    #   it "should crawl exactly 20" do
+    #     crawl = @cobweb.start("http://yepadeperrors.wordpress.com/")
+    #     @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+    #     wait_for_crawl_finished crawl[:crawl_id]
+    #     @redis.get("crawl_job_enqueued_count").to_i.should == 20
+    #   end
+    # 
+    # end
     describe "limit to 1" do
       before(:each) do
         @request[:crawl_limit] = 1
@@ -245,12 +261,17 @@ def running?(crawl_id)
     result = true
   else
     if status == @last_stat
-      if @counter > 5
+      if @counter > 20
+        puts ""
         raise "Static status: #{status}"
       else
         @counter += 1
       end
-      puts "Static Status.. #{6-@counter}"
+      if @counter == 1
+        print "Static Status.. #{21-@counter}"
+      else
+        print ".#{21-@counter}"
+      end
     else
       result = status != CobwebCrawlHelper::FINISHED && status != CobwebCrawlHelper::CANCELLED
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.74
+  version: 0.0.75
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-15 00:00:00.000000000 Z
+date: 2012-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70347429190520 !ruby/object:Gem::Requirement
+  requirement: &70303208832100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429190520
+  version_requirements: *70303208832100
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70347429190020 !ruby/object:Gem::Requirement
+  requirement: &70303208831180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429190020
+  version_requirements: *70303208831180
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70347429189540 !ruby/object:Gem::Requirement
+  requirement: &70303208830080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429189540
+  version_requirements: *70303208830080
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70347429188880 !ruby/object:Gem::Requirement
+  requirement: &70303208829280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429188880
+  version_requirements: *70303208829280
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70347429187340 !ruby/object:Gem::Requirement
+  requirement: &70303208828000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429187340
+  version_requirements: *70303208828000
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70347429185820 !ruby/object:Gem::Requirement
+  requirement: &70303208826740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429185820
+  version_requirements: *70303208826740
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70347429185040 !ruby/object:Gem::Requirement
+  requirement: &70303208825020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429185040
+  version_requirements: *70303208825020
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70347429184340 !ruby/object:Gem::Requirement
+  requirement: &70303208823900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429184340
+  version_requirements: *70303208823900
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70347429183120 !ruby/object:Gem::Requirement
+  requirement: &70303208822980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429183120
+  version_requirements: *70303208822980
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70347429181840 !ruby/object:Gem::Requirement
+  requirement: &70303208821840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70347429181840
+  version_requirements: *70303208821840
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70347429180860 !ruby/object:Gem::Requirement
+  requirement: &70303208821320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,7 +131,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70347429180860
+  version_requirements: *70303208821320
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface