cobweb 0.0.73 → 0.0.74

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.73
+h1. Cobweb v0.0.74
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 

data/lib/cobweb.rb

@@ -4,7 +4,6 @@ require 'resque'
 require "addressable/uri"
 require 'digest/sha1'
 require 'base64'
-require 'namespaced_redis'
 
 Dir[File.dirname(__FILE__) + '/**/*.rb'].each do |file|
   require file
@@ -46,6 +45,7 @@ class Cobweb
     default_text_mime_types_to                ["text/*", "application/xhtml+xml"]
     default_obey_robots_to                    false
     default_user_agent_to                     "cobweb/#{Cobweb.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"
+    default_valid_mime_types_to                ["*/*"]
     
   end
   
@@ -65,7 +65,7 @@ class Cobweb
     end
     
     request.merge!(@options)
-    @redis = NamespacedRedis.new(request[:redis_options], "cobweb-#{Cobweb.version}-#{request[:crawl_id]}")
+    @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{request[:crawl_id]}", :redis => Redis.new(request[:redis_options]))
     @redis.set("original_base_url", base_url)
     @redis.hset "statistics", "queued_at", DateTime.now
     @redis.set("crawl-counter", 0)
@@ -110,9 +110,9 @@ class Cobweb
     
     # connect to redis
     if options.has_key? :crawl_id
-      redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
+      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{options[:crawl_id]}", :redis => Redis.new(@options[:redis_options]))
     else
-      redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}")
+      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => Redis.new(@options[:redis_options]))
     end
 
     content = {:base_url => url}
@@ -269,9 +269,9 @@ class Cobweb
     
     # connect to redis
     if options.has_key? :crawl_id
-      redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
+      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{options[:crawl_id]}", :redis => Redis.new(@options[:redis_options]))
     else
-      redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}")
+      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => Redis.new(@options[:redis_options]))
     end
     
     content = {:base_url => url}

data/lib/cobweb_crawl_helper.rb

@@ -15,7 +15,7 @@ class CobwebCrawlHelper
     @stats = Stats.new(data)
   end
   
-  def destroy(options)
+  def destroy(options={})
     
     options[:queue_name] = "cobweb_crawl_job" unless options.has_key?(:queue_name)
     options[:finished_resque_queue] = CobwebFinishedJob unless options.has_key?(:finished_resque_queue)

data/lib/cobweb_crawler.rb

@@ -1,7 +1,7 @@
 require 'digest/md5'
 require 'date'
 require 'ap'
-#require 'namespaced_redis'
+require 'redis-namespace'
 
 # CobwebCrawler is a standalone crawler, it includes a built in statistics monitor using Sinatra.
 class CobwebCrawler
@@ -20,7 +20,7 @@ class CobwebCrawler
       @options[:crawl_id] = @crawl_id
     end
     
-    @redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}-#{@crawl_id}")
+    @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@crawl_id}", :redis => Redis.new(@options[:redis_options]))
     @options[:internal_urls] = [] if @options[:internal_urls].nil?
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
     @debug = @options[:debug]

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.73"
+    "0.0.74"
   end
 
 end

data/lib/content_link_parser.rb

@@ -6,7 +6,7 @@ class ContentLinkParser
 
   # Parses the content and absolutizes the urls based on url.  Options can be setup to determine the links that are extracted.
   def initialize(url, content, options = {})
-    @options = options
+    @options = {}.merge(options)
     @url = url
     @doc = Nokogiri::HTML(content)
     
@@ -61,8 +61,7 @@ class ContentLinkParser
       end
       links.uniq
     else
-      puts "Warning: There was no configuration on how to find #{m} links"
-      []
+      super
     end
   end
   

data/lib/crawl.rb

@@ -0,0 +1,263 @@
+module CobwebModule
+  class Crawl
+    
+    def initialize(options={})
+      @options = HashUtil.deep_symbolize_keys(options)
+      
+      setup_defaults
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@options[:crawl_id]}", Redis.new(@options[:redis_options]))
+      @stats = Stats.new(@options)
+      @debug = @options[:debug]
+      @first_to_finish = false
+      
+    end
+    
+    # Returns true if the url requested is already in the crawled queue
+    def already_crawled?(link=@options[:url])
+       @redis.sismember "crawled", link
+    end
+    
+    def already_queued?(link)
+      @redis.sismember "queued", link
+    end
+    
+    # Returns true if the crawl count is within limits
+    def within_crawl_limits?
+      @options[:crawl_limit].nil? || crawl_counter < @options[:crawl_limit].to_i
+    end
+
+    # Returns true if the processed count is within limits
+    def within_process_limits?
+      @options[:crawl_limit].nil? || process_counter < @options[:crawl_limit].to_i
+    end
+
+    # Returns true if the queue count is calculated to be still within limits when complete
+    def within_queue_limits?
+      
+      # if we are limiting by page we can't limit the queue size as we don't know the mime type until retrieved
+      if @options[:crawl_limit_by_page]
+        return true
+        
+      # if a crawl limit is set, limit queue size to crawled + queue
+      elsif @options[:crawl_limit].to_i > 0
+        (queue_counter + crawl_counter) < @options[:crawl_limit].to_i
+      
+      # no crawl limit set so always within queue limit
+      else
+        true
+      end
+    end
+    
+    def retrieve
+      unless already_crawled?
+        if within_crawl_limits?
+          @stats.update_status("Retrieving #{@options[:url]}...")
+          @content = Cobweb.new(@options).get(@options[:url], @options)
+          if @options[:url] == @redis.get("original_base_url")
+             @redis.set("crawled_base_url", @content[:base_url])
+          end
+          update_queues
+      
+          if content.permitted_type?
+            ## update statistics
+          
+            @stats.update_statistics(@content)
+            return true
+          end
+        else
+          decrement_queue_counter
+        end
+      else
+        decrement_queue_counter
+      end
+      false
+    end
+    
+    def process_links &block
+      
+      # set the base url if this is the first page
+      set_base_url @redis
+      
+      @cobweb_links = CobwebLinks.new(@options)
+      if within_queue_limits?
+        internal_links = ContentLinkParser.new(@options[:url], content.body, @options).all_links(:valid_schemes => [:http, :https])
+        #get rid of duplicate links in the same page.
+        internal_links.uniq!
+        # select the link if its internal
+        internal_links.select! { |link| @cobweb_links.internal?(link) }
+
+        # reject the link if we've crawled it or queued it
+        internal_links.reject! { |link| @redis.sismember("crawled", link) }
+        internal_links.reject! { |link| @redis.sismember("queued", link) }
+
+        internal_links.each do |link|
+          if within_queue_limits? && !already_queued?(link) && !already_crawled?(link)
+            if status != CobwebCrawlHelper::CANCELLED
+              yield link if block_given?
+              unless link.nil?
+                @redis.sadd "queued", link
+                increment_queue_counter
+              end
+            else
+              puts "Cannot enqueue new content as crawl has been cancelled." if @options[:debug]
+            end
+          end
+        end
+      end
+    end
+    
+    def content
+      raise "Content is not available" if @content.nil?
+      CobwebModule::CrawlObject.new(@content, @options) 
+    end
+    
+    def update_queues
+      @redis.multi do
+        #@redis.incr "inprogress"
+        # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
+        @redis.srem "queued", @options[:url]
+        @redis.sadd "crawled", @options[:url]
+        if content.url != @options[:url]
+          @redis.srem "queued", content.url
+          @redis.sadd "crawled", content.url
+        end
+        # increment the counter if we are not limiting by page only || we are limiting count by page and it is a page
+        if @options[:crawl_limit_by_page]
+          ap "#{content.mime_type} - #{content.url}"
+          if content.mime_type.match("text/html")
+            increment_crawl_counter
+          end
+        else
+          increment_crawl_counter
+        end
+        decrement_queue_counter
+      end
+    end
+    
+    def to_be_processed?
+      !finished? || first_to_finish? || within_process_limits?
+    end
+    
+    def process
+      if @options[:crawl_limit_by_page]
+        if content.mime_type.match("text/html")
+          increment_process_counter
+        end
+      else
+        increment_process_counter
+      end
+    end
+    
+    def finished?
+      print_counters
+      # if there's nothing left queued or the crawled limit has been reached
+      if @options[:crawl_limit].nil? || @options[:crawl_limit] == 0
+        if queue_counter.to_i == 0
+          finished
+          return true
+        end
+      elsif (queue_counter.to_i) == 0 || crawl_counter.to_i >= @options[:crawl_limit].to_i
+        finished
+        return true
+      end
+      false
+    end
+    
+    def finished
+      set_first_to_finish if !@redis.exists("first_to_finish")
+      ap "CRAWL FINISHED  #{@options[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if @options[:debug]
+      @stats.end_crawl(@options)
+    end
+    
+    def set_first_to_finish
+      @redis.watch("first_to_finish") do
+        if !@redis.exists("first_to_finish")
+          @redis.multi do
+            puts "set first to finish"
+            @first_to_finish = true
+            @redis.set("first_to_finish", 1)
+          end
+        else
+          @redis.unwatch
+        end
+      end
+    end
+    
+    
+    def first_to_finish? 
+      @first_to_finish
+    end
+    
+    def crawled_base_url
+      @redis.get("crawled_base_url")
+    end
+    
+    def statistics
+      @stats.get_statistics
+    end
+    
+    def redis
+      @redis
+    end
+    
+    private
+    def setup_defaults
+      @options[:redis_options] = {} unless @options.has_key? :redis_options
+      @options[:crawl_limit_by_page] = false unless @options.has_key? :crawl_limit_by_page
+      @options[:valid_mime_types] = ["*/*"] unless @options.has_key? :valid_mime_types
+    end
+    
+    # Increments the queue counter and refreshes crawl counters
+    def increment_queue_counter
+      @redis.incr "queue-counter"
+    end
+    # Increments the crawl counter and refreshes crawl counters
+    def increment_crawl_counter
+      @redis.incr "crawl-counter"
+    end
+    # Increments the process counter
+    def increment_process_counter
+      @redis.incr "process-counter"
+    end
+    # Decrements the queue counter and refreshes crawl counters
+    def decrement_queue_counter
+      @redis.decr "queue-counter"
+    end
+    
+    def crawl_counter
+      @redis.get("crawl-counter").to_i
+    end
+    def queue_counter
+      @redis.get("queue-counter").to_i
+    end
+    def process_counter
+      @redis.get("process-counter").to_i
+    end
+    
+    def status
+      @stats.get_status
+    end
+    
+    def print_counters
+      puts counters
+    end
+    
+    def counters
+      "crawl_counter: #{crawl_counter} queue_counter: #{queue_counter} process_counter: #{process_counter} crawl_limit: #{@options[:crawl_limit]}"
+    end
+    
+    # Sets the base url in redis.  If the first page is a redirect, it sets the base_url to the destination
+    def set_base_url(redis)
+      if redis.get("base_url").nil?
+        unless !defined?(content.redirect_through) || content.redirect_through.empty? || !@options[:first_page_redirect_internal]
+          uri = Addressable::URI.parse(content.redirect_through.last)
+          redis.sadd("internal_urls", [uri.scheme, "://", uri.host, "/*"].join)
+        end
+        redis.set("base_url", content.url)
+      end
+    end
+
+
+    
+  end
+end

data/lib/crawl_job.rb

@@ -5,132 +5,58 @@ class CrawlJob
   require "net/https"  
   require "uri"
   require "redis"
-  require 'namespaced_redis'
-
+  
   @queue = :cobweb_crawl_job
-
+  
   # Resque perform method to maintain the crawl, enqueue found links and detect the end of crawl
   def self.perform(content_request)
-    # change all hash keys to symbols
-    content_request = HashUtil.deep_symbolize_keys(content_request)
-    @content_request = content_request
-    @crawl = CobwebCrawlHelper.new(content_request)
-    
-    content_request[:redis_options] = {} unless content_request.has_key? :redis_options
-    content_request[:crawl_limit_by_page] = false unless content_request.has_key? :crawl_limit_by_page
-    content_request[:valid_mime_types] = ["*/*"] unless content_request.has_key? :valid_mime_types
     
-    @redis = NamespacedRedis.new(content_request[:redis_options], "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
-    @stats = Stats.new(content_request)
+    # setup the crawl class to manage the crawl of this object
+    @crawl = CobwebModule::Crawl.new(content_request)
     
-    @debug = content_request[:debug]
+    # update the counters and then perform the get, returns false if we are outwith limits
+    if @crawl.retrieve
     
-    # check we haven't crawled this url before
-    unless @redis.sismember "crawled", content_request[:url]
-      # if there is no limit or we're still under it lets get the url
-      if within_crawl_limits?(content_request[:crawl_limit])
-        if @crawl.status != CobwebCrawlHelper::CANCELLED
-          content = Cobweb.new(content_request).get(content_request[:url], content_request)
-          if content_request[:url] == @redis.get("original_base_url")
-             @redis.set("crawled_base_url", content[:base_url])
-          end
-          if is_permitted_type(content)
-            begin
-              @redis.incr "inprogress"
-              # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
-              @redis.srem "queued", content_request[:url]
-              @redis.sadd "crawled", content_request[:url]
-              @redis.srem "queued", content[:url]
-              @redis.sadd "crawled", content[:url]
-              # increment the counter if we are not limiting by page only || we are limiting count by page and it is a page
-              if content_request[:crawl_limit_by_page]
-                if content[:mime_type].match("text/html")
-                  increment_crawl_started_counter
-                end
-              else
-                increment_crawl_started_counter
-              end
-
-              ## update statistics
-              @stats.update_status("Crawling #{content_request[:url]}...")
-              @stats.update_statistics(content)
-
-              # set the base url if this is the first page
-              set_base_url @redis, content, content_request
-            
-              @cobweb_links = CobwebLinks.new(content_request)
-              if within_queue_limits?(content_request[:crawl_limit])
-                internal_links = ContentLinkParser.new(content_request[:url], content[:body], content_request).all_links(:valid_schemes => [:http, :https])
-                #get rid of duplicate links in the same page.
-                internal_links.uniq!
-                # select the link if its internal
-                internal_links.select! { |link| @cobweb_links.internal?(link) }
-
-                # reject the link if we've crawled it or queued it
-                internal_links.reject! { |link| @redis.sismember("crawled", link) }
-                internal_links.reject! { |link| @redis.sismember("queued", link) }
-
-                internal_links.each do |link|
-                  puts link
-                  puts "Not enqueuing due to cancelled crawl" if @crawl.status == CobwebCrawlHelper::CANCELLED
-                  if within_queue_limits?(content_request[:crawl_limit])
-                    if @crawl.status != CobwebCrawlHelper::CANCELLED
-                      enqueue_content(content_request, link) 
-                    else
-                      puts "Cannot enqueue new content as crawl has been cancelled." if content_request[:debug]
-                    end
-                  end
-                end
-              end
-
-              # enqueue to processing queue
-              send_to_processing_queue(content, content_request)
+      # if the crawled object is an object type we are interested
+      if @crawl.content.permitted_type?
+        
+        # extract links from content and process them if we are still within queue limits (block will not run if we are outwith limits)
+        @crawl.process_links do |link|
 
-              #if the enqueue counter has been requested update that
-              if content_request.has_key?(:enqueue_counter_key)
-                enqueue_redis = NamespacedRedis.new(content_request[:redis_options], content_request[:enqueue_counter_namespace].to_s)
-                current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
-                enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
-              end
+          # enqueue the links to resque
+          puts "ENQUEUED LINK: #{link}"
+          enqueue_content(content_request, link) 
 
-            ensure
-              @redis.decr "inprogress"
-              #update the queued and crawled lists if we are within the crawl limits.
-
-              # update the queue and crawl counts -- doing this very late in the piece so that the following transaction all occurs at once.
-              # really we should do this with a lock https://github.com/PatrickTulskie/redis-lock
-              # stewart: i'm looking at the layout of this, think that there is scope for cleaning up the perform method to be more DRY.
-              if content_request[:crawl_limit_by_page]
-                if content[:mime_type].match("text/html")
-                  increment_crawl_counter
-                end
-              else
-                increment_crawl_counter
-              end
-              puts "Crawled: #{crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{queue_counter} In Progress: #{crawl_started_counter-crawl_counter}" if @debug
-            end
-          else
-            puts "ignoring #{content_request[:url]} as mime_type is #{content[:mime_type]}" if content_request[:debug]
+        end
+        
+        
+        if @crawl.to_be_processed?
+          @crawl.process
+          
+          # enqueue to processing queue
+          @crawl.redis.incr("crawl_job_enqueued_count")
+          puts "ENQUEUED [#{@crawl.redis.get("crawl_job_enqueued_count")}] URL: #{@crawl.content.url}"
+          send_to_processing_queue(@crawl.content.to_hash, content_request)
+          
+          
+          #if the enqueue counter has been requested update that
+          if content_request.has_key?(:enqueue_counter_key)
+            enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => Redis.new(content_request[:redis_options]))
+            current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
+            enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
           end
         else
-          puts "ignoring #{content_request[:url]} as crawl has been cancelled." if content_request[:debug]
+          ap "@crawl.finished? #{@crawl.finished?}"
+          ap "@crawl.within_crawl_limits? #{@crawl.within_crawl_limits?}"
+          ap "@crawl.first_to_finish? #{@crawl.first_to_finish?}"
         end
-      else
-        puts "ignoring #{content_request[:url]} as outside of crawl limits." if content_request[:debug]
       end
-      
-    else
-      @redis.srem "queued", content_request[:url]
-      puts "Already crawled #{content_request[:url]}" if content_request[:debug]
     end
-    
-    decrement_queue_counter
-    # if there's nothing left queued or the crawled limit has been reached
-    if content_request[:crawl_limit].nil? || content_request[:crawl_limit] == 0
-      if queue_counter + crawl_started_counter - crawl_counter == 0
-        finished(content_request)
-      end
-    elsif (queue_counter+crawl_started_counter-crawl_counter)== 0 || crawl_counter >= content_request[:crawl_limit].to_i
+
+    # test queue and crawl sizes to see if we have completed the crawl
+    ap "finished? #{@crawl.finished?}"
+    ap "first_to_finish? #{@crawl.first_to_finish?}" if @crawl.finished?
+    if @crawl.finished? && @crawl.first_to_finish?
       finished(content_request)
     end
     
@@ -138,19 +64,12 @@ class CrawlJob
 
   # Sets the crawl status to CobwebCrawlHelper::FINISHED and enqueues the crawl finished job
   def self.finished(content_request)
-    # finished
-    if @crawl.status != CobwebCrawlHelper::FINISHED and @crawl.status != CobwebCrawlHelper::CANCELLED && @redis.get("inprogress").to_i==0
-      ap "CRAWL FINISHED  #{content_request[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if content_request[:debug]
-      @stats.end_crawl(content_request)
-      
-      additional_stats = {:crawl_id => content_request[:crawl_id], :crawled_base_url => @redis.get("crawled_base_url")}
-      additional_stats[:redis_options] = content_request[:redis_options] unless content_request[:redis_options] == {}
-      additional_stats[:source_id] = content_request[:source_id] unless content_request[:source_id].nil?
-      
-      Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge(additional_stats))
-    else
-      # nothing to report here, we're skipping the remaining urls as we're outside of the crawl limit
-    end
+    additional_stats = {:crawl_id => content_request[:crawl_id], :crawled_base_url => @crawl.crawled_base_url}
+    additional_stats[:redis_options] = content_request[:redis_options] unless content_request[:redis_options] == {}
+    additional_stats[:source_id] = content_request[:source_id] unless content_request[:source_id].nil?
+    
+    @crawl.redis.incr("crawl_finished_enqueued_count")
+    Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @crawl.statistics.merge(additional_stats))
   end
   
   # Enqueues the content to the processing queue setup in options
@@ -171,34 +90,6 @@ class CrawlJob
 
   private
   
-  # Helper method to determine if this content is to be processed or not
-  def self.is_permitted_type(content)
-    @content_request[:valid_mime_types].each do |mime_type|
-      return true if content[:mime_type].match(Cobweb.escape_pattern_for_regex(mime_type))
-    end
-    false
-  end
-  
-  # Returns true if the crawl count is within limits
-  def self.within_crawl_limits?(crawl_limit)
-    crawl_limit.nil? or crawl_counter < crawl_limit.to_i
-  end
-  
-  # Returns true if the queue count is calculated to be still within limits when complete
-  def self.within_queue_limits?(crawl_limit)
-    (@content_request[:crawl_limit_by_page]&& (crawl_limit.nil? or crawl_counter < crawl_limit.to_i)) || within_crawl_limits?(crawl_limit) && (crawl_limit.nil? || (queue_counter + crawl_counter) < crawl_limit.to_i)
-  end
-  
-  # Sets the base url in redis.  If the first page is a redirect, it sets the base_url to the destination
-  def self.set_base_url(redis, content, content_request)
-    if redis.get("base_url").nil?
-      unless content[:redirect_through].nil? || content[:redirect_through].empty? || !content_request[:first_page_redirect_internal]
-        uri = Addressable::URI.parse(content[:redirect_through].last)
-        redis.sadd("internal_urls", [uri.scheme, "://", uri.host, "/*"].join)
-      end
-      redis.set("base_url", content[:url])
-    end
-  end
   
   # Enqueues content to the crawl_job queue
   def self.enqueue_content(content_request, link)
@@ -206,43 +97,8 @@ class CrawlJob
     new_request[:url] = link
     new_request[:parent] = content_request[:url]
     #to help prevent accidentally double processing a link, let's mark it as queued just before the Resque.enqueue statement, rather than just after.
-    @redis.sadd "queued", link
     Resque.enqueue(CrawlJob, new_request)
-    increment_queue_counter
   end
   
-  # Increments the queue counter and refreshes crawl counters
-  def self.increment_queue_counter
-    @redis.incr "queue-counter"
-  end
-  # Increments the crawl counter and refreshes crawl counters
-  def self.increment_crawl_counter
-    @redis.incr "crawl-counter"
-  end
-  def self.increment_crawl_started_counter
-    @redis.incr "crawl-started-counter"
-  end
-  # Decrements the queue counter and refreshes crawl counters
-  def self.decrement_queue_counter
-    @redis.decr "queue-counter"
-  end
-  
-  def self.crawl_counter
-    @redis.get("crawl-counter").to_i
-  end
-  def self.crawl_started_counter
-    @redis.get("crawl-started-counter").to_i
-  end
-  def self.queue_counter
-    @redis.get("queue-counter").to_i
-  end
-  
-  def self.print_counters
-    puts counters
-  end
-
-  def self.counters
-    "crawl_counter: #{crawl_counter} crawl_started_counter: #{crawl_started_counter} queue_counter: #{queue_counter}"
-  end
 
 end

data/lib/crawl_object.rb

@@ -0,0 +1,30 @@
+module CobwebModule
+  class CrawlObject
+    
+    def initialize(content_hash, options={})
+      @content = HashUtil.deep_symbolize_keys(content_hash)
+      @options = options
+    end
+    
+    
+    # Helper method to determine if this content is to be processed or not
+    def permitted_type?
+      @options[:valid_mime_types].each do |valid_mime_type|
+        return true if @content[:mime_type].match(Cobweb.escape_pattern_for_regex(valid_mime_type))
+      end
+      false
+    end
+    
+    def method_missing(m)
+      if @content.keys.include? m.to_sym
+        @content[m.to_sym]
+      else
+        super
+      end
+    end
+    
+    def to_hash
+      @content
+    end
+  end
+end

data/lib/hash_util.rb

@@ -3,6 +3,7 @@ class HashUtil
   
   # Returns a hash with the keys converted to symbols
   def self.deep_symbolize_keys(hash)
+    raise "Cannot symbolize keys for a nil object" if hash.nil?
     hash.keys.each do |key|
       value = hash[key]
       hash.delete(key)

data/lib/server.rb

@@ -16,7 +16,7 @@ class Server < Sinatra::Base
     @crawls = []
     @full_redis.smembers("cobweb_crawls").each do |crawl_id|      
       version = cobweb_version(crawl_id)
-      redis = NamespacedRedis.new(redis_options, "cobweb-#{version}-#{crawl_id}")
+      redis = Redis::Namespace.new("cobweb-#{version}-#{crawl_id}", :redis => Redis.new(redis_options))
       stats = HashUtil.deep_symbolize_keys({
         :cobweb_version => version,
         :crawl_details => redis.hgetall("crawl_details"),
@@ -33,7 +33,7 @@ class Server < Sinatra::Base
   get '/statistics/:crawl_id' do
     
     version = cobweb_version(params[:crawl_id])
-    redis = NamespacedRedis.new(redis_options, "cobweb-#{version}-#{params[:crawl_id]}")
+    redis = Redis::Namespace.new("cobweb-#{version}-#{params[:crawl_id]}", :redis => Redis.new(redis_options))
     
     @statistics = HashUtil.deep_symbolize_keys(redis.hgetall("statistics"))
     if @statistics[:status_counts].nil?

data/lib/stats.rb

@@ -8,7 +8,7 @@ class Stats
   def initialize(options)
     options[:redis_options] = {} unless options.has_key? :redis_options
     @full_redis = Redis.new(options[:redis_options])
-    @redis = NamespacedRedis.new(options[:redis_options], "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
+    @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{options[:crawl_id]}", :redis => @full_redis)
   end
   
   # Sets up the crawl in statistics

data/spec/cobweb/cobweb_job_spec.rb

@@ -9,7 +9,7 @@ describe Cobweb, :local_only => true do
     # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES
     puts "Starting Workers... Please Wait..."
     `mkdir log`
-    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=1 QUEUE=cobweb_crawl_job > log/output.log &")
+    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=3 QUEUE=cobweb_crawl_job > log/output.log &")
     puts "Workers Started."
 
   end
@@ -17,6 +17,7 @@ describe Cobweb, :local_only => true do
   before(:each) do
     @base_url = "http://localhost:3532/"
     @base_page_count = 77
+
     clear_queues
   end
 
@@ -29,6 +30,7 @@ describe Cobweb, :local_only => true do
         :debug => false,
         :cache => nil
       }
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", Redis.new)
       @cobweb = Cobweb.new @request
     end
     it "should not crawl anything if nothing has started" do
@@ -37,7 +39,7 @@ describe Cobweb, :local_only => true do
       crawl_obj.destroy
       @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
       wait_for_crawl_finished crawl[:crawl_id]
-      Resque.size("cobweb_process_job").should == 0
+      @redis.get("crawl_job_enqueued_count").to_i.should == 0
     end
 
     it "should not complete the crawl when cancelled" do
@@ -47,8 +49,8 @@ describe Cobweb, :local_only => true do
       crawl_obj.destroy
       @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
       wait_for_crawl_finished crawl[:crawl_id]
-      Resque.size("cobweb_process_job").should > 0
-      Resque.size("cobweb_process_job").should_not == @base_page_count
+      @redis.get("crawl_job_enqueued_count").to_i.should > 0
+      @redis.get("crawl_job_enqueued_count").to_i.should_not == @base_page_count
     end
 
   end
@@ -61,22 +63,24 @@ describe Cobweb, :local_only => true do
         :debug => false,
         :cache => nil
       }
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", Redis.new)
+
       @cobweb = Cobweb.new @request
     end
 
     it "should crawl entire site" do
-      ap Resque.size("cobweb_process_job")
       crawl = @cobweb.start(@base_url)
       @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
       wait_for_crawl_finished crawl[:crawl_id]
-      ap @stat.get_statistics
-      Resque.size("cobweb_process_job").should == @base_page_count
+      @redis.get("crawl_job_enqueued_count").to_i.should == @base_page_count
+      @redis.get("crawl_finished_enqueued_count").to_i.should == 1
     end
     it "detect crawl finished once" do
       crawl = @cobweb.start(@base_url)
       @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
       wait_for_crawl_finished crawl[:crawl_id]
-      Resque.size("cobweb_finished_job").should == 1
+      @redis.get("crawl_job_enqueued_count").to_i.should == @base_page_count
+      @redis.get("crawl_finished_enqueued_count").to_i.should == 1
     end
   end
   describe "with limited mime_types" do
@@ -87,6 +91,7 @@ describe Cobweb, :local_only => true do
         :cache => nil,
         :valid_mime_types => ["text/html"]
       }
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", Redis.new)
       @cobweb = Cobweb.new @request
     end
 
@@ -94,7 +99,7 @@ describe Cobweb, :local_only => true do
       crawl = @cobweb.start(@base_url)
       @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
       wait_for_crawl_finished crawl[:crawl_id]
-      Resque.size("cobweb_process_job").should == 8
+      @redis.get("crawl_job_enqueued_count").to_i.should == 8
 
       mime_types = Resque.peek("cobweb_process_job", 0, 100).map{|job| job["args"][0]["mime_type"]}
       mime_types.count.should == 8
@@ -110,6 +115,7 @@ describe Cobweb, :local_only => true do
         :quiet => true,
         :cache => nil
       }
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", Redis.new)
     end
 
     describe "limit to 1" do
@@ -122,19 +128,19 @@ describe Cobweb, :local_only => true do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_process_job").should_not == @base_page_count
+        @redis.get("crawl_job_enqueued_count").to_i.should_not == @base_page_count
       end
       it "should only crawl 1 page" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_process_job").should == 1
+        @redis.get("crawl_job_enqueued_count").to_i.should == 1
       end
       it "should notify of crawl finished once" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_finished_job").should == 1
+        @redis.get("crawl_finished_enqueued_count").to_i.should == 1
       end
     end
 
@@ -145,6 +151,7 @@ describe Cobweb, :local_only => true do
         @cobweb = Cobweb.new @request
       end
 
+      # the following describes when we want all the assets of a page, and the page itself, but we only want 5 pages
       it "should only use html pages towards the crawl limit" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
@@ -165,19 +172,19 @@ describe Cobweb, :local_only => true do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_process_job").should_not == @base_page_count
+        @redis.get("crawl_job_enqueued_count").to_i.should_not == @base_page_count
       end
       it "should notify of crawl finished once" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_finished_job").should == 1
+        @redis.get("crawl_finished_enqueued_count").to_i.should == 1
       end
       it "should only crawl 10 objects" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_process_job").should == 10
+        @redis.get("crawl_job_enqueued_count").to_i.should == 10
       end
     end
 
@@ -191,23 +198,24 @@ describe Cobweb, :local_only => true do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_process_job").should == @base_page_count
+        @redis.get("crawl_job_enqueued_count").to_i.should == @base_page_count
       end
       it "should notify of crawl finished once" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_finished_job").should == 1
+        @redis.get("crawl_finished_enqueued_count").to_i.should == 1
       end
       it "should not crawl 100 pages" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_process_job").should_not == 100
+        @redis.get("crawl_job_enqueued_count").to_i.should_not == 100
       end
     end
   end
 
+
   after(:all) do
 
     @all_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
@@ -220,26 +228,43 @@ describe Cobweb, :local_only => true do
 end
 
 def wait_for_crawl_finished(crawl_id, timeout=20)
-  counter = 0
+  @counter = 0
   start_time = Time.now
   while(running?(crawl_id) && Time.now < start_time + timeout) do
-      sleep 0.5
-    end
-    if Time.now > start_time + timeout
-      raise "End of crawl not detected"
-    end
+    sleep 0.5
   end
-
-  def running?(crawl_id)
-    @stat.get_status != CobwebCrawlHelper::FINISHED and @stat.get_status != CobwebCrawlHelper::CANCELLED
+  if Time.now > start_time + timeout
+    raise "End of crawl not detected"
   end
+end
 
-  def clear_queues
-    Resque.queues.each do |queue|
-      Resque.remove_queue(queue)
+def running?(crawl_id)
+  status = @stat.get_status
+  result = true
+  if status == CobwebCrawlHelper::STARTING
+    result = true
+  else
+    if status == @last_stat
+      if @counter > 5
+        raise "Static status: #{status}"
+      else
+        @counter += 1
+      end
+      puts "Static Status.. #{6-@counter}"
+    else
+      result = status != CobwebCrawlHelper::FINISHED && status != CobwebCrawlHelper::CANCELLED
     end
+  end
+  @last_stat = @stat.get_status
+  result
+end
 
-    Resque.size("cobweb_process_job").should == 0
-    Resque.size("cobweb_finished_job").should == 0
-    Resque.peek("cobweb_process_job", 0, 200).should be_empty
+def clear_queues
+  Resque.queues.each do |queue|
+    Resque.remove_queue(queue)
   end
+
+  Resque.size("cobweb_process_job").should == 0
+  Resque.size("cobweb_finished_job").should == 0
+  Resque.peek("cobweb_process_job", 0, 200).should be_empty
+end

data/spec/cobweb/content_link_parser_spec.rb

@@ -76,11 +76,9 @@ describe ContentLinkParser do
           links.length.should == 3
         end
       end
-      describe "returning unknown link type" do
+      describe "returning unknown link type should raise an error" do
         it "should return an empty array" do
-          links = @content_parser.asdfasdfsadf
-          links.should_not be_nil
-          links.should be_an_instance_of Array
+          lambda {@content_parser.asdfasdfsadf}.should raise_error
         end
       end
     end
@@ -122,7 +120,7 @@ describe ContentLinkParser do
     describe "ignoring default tags" do
       it "should not return any links" do
         parser = ContentLinkParser.new("http://sample-links.com", @content, :ignore_default_tags => true)
-        parser.links.should be_empty
+        lambda{parser.links}.should raise_error(NoMethodError)
       end
     end
   end

data/spec/cobweb/site_test_spec.rb.tmp

@@ -0,0 +1,101 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Cobweb, :local_only => true do
+
+  before(:all) do
+    #store all existing resque process ids so we don't kill them afterwards
+    @existing_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
+
+    # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES
+    puts "Starting Workers... Please Wait..."
+    `mkdir log`
+    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=3 QUEUE=cobweb_crawl_job > log/output.log &")
+    puts "Workers Started."
+
+  end
+
+  before(:each) do
+    @base_url = "http://localhost:3532/"
+    @base_page_count = 77
+    clear_queues
+  end
+
+  describe "with a crawl limit" do
+    before(:each) do
+      @request = {
+        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+        :quiet => true,
+        :cache => nil,
+        :use_encoding_safe_process_job => true,
+        :crawl_limit_by_page => true
+      }
+    end
+    
+    describe "on ancestry.com.au" do
+      describe "limited to 100" do
+        before(:each) do
+          @request[:crawl_limit] = 100
+          @request[:valid_mime_types] = ["text/html"]
+          @cobweb = Cobweb.new @request        
+        end
+        
+        it "should crawl 100 pages" do
+          crawl = @cobweb.start("http://www.ancestry.com.au/")
+          @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+          wait_for_crawl_finished crawl[:crawl_id], 180
+          puts "Crawled #{Resque.size("encoding_safe_process_job")} pages."
+        end
+      end
+      
+      describe "limited to 999" do
+        before(:each) do
+          @request[:crawl_limit] = 999
+          @cobweb = Cobweb.new @request        
+        end
+      
+        it "should crawl 999 pages" do
+          crawl = @cobweb.start("http://www.ancestry.com.au/")
+          @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+          wait_for_crawl_finished crawl[:crawl_id], 720
+          puts "Crawled #{Resque.size("encoding_safe_process_job")} pages."
+        end
+      end
+    __END__
+    
+  end
+
+  after(:all) do
+
+    @all_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
+    command = "kill -9 #{(@all_processes - @existing_processes).join(" ")}"
+    IO.popen(command)
+
+    clear_queues
+  end
+
+end
+
+def wait_for_crawl_finished(crawl_id, timeout=20)
+  counter = 0
+  start_time = Time.now
+  while(running?(crawl_id) && Time.now < start_time + timeout) do
+      sleep 0.5
+    end
+    if Time.now > start_time + timeout
+      raise "End of crawl not detected"
+    end
+  end
+
+  def running?(crawl_id)
+    @stat.get_status != CobwebCrawlHelper::FINISHED and @stat.get_status != CobwebCrawlHelper::CANCELLED
+  end
+
+  def clear_queues
+    Resque.queues.each do |queue|
+      Resque.remove_queue(queue)
+    end
+
+    Resque.size("cobweb_process_job").should == 0
+    Resque.size("cobweb_finished_job").should == 0
+    Resque.peek("cobweb_process_job", 0, 200).should be_empty
+  end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.73
+  version: 0.0.74
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-20 00:00:00.000000000 Z
+date: 2012-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70226914018080 !ruby/object:Gem::Requirement
+  requirement: &70347429190520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914018080
+  version_requirements: *70347429190520
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70226914017080 !ruby/object:Gem::Requirement
+  requirement: &70347429190020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914017080
+  version_requirements: *70347429190020
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70226914016400 !ruby/object:Gem::Requirement
+  requirement: &70347429189540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914016400
+  version_requirements: *70347429189540
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70226914015220 !ruby/object:Gem::Requirement
+  requirement: &70347429188880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914015220
+  version_requirements: *70347429188880
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70226914014640 !ruby/object:Gem::Requirement
+  requirement: &70347429187340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914014640
+  version_requirements: *70347429187340
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70226914013860 !ruby/object:Gem::Requirement
+  requirement: &70347429185820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914013860
+  version_requirements: *70347429185820
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70226914013140 !ruby/object:Gem::Requirement
+  requirement: &70347429185040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914013140
+  version_requirements: *70347429185040
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70226914012280 !ruby/object:Gem::Requirement
+  requirement: &70347429184340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914012280
+  version_requirements: *70347429184340
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70226914011460 !ruby/object:Gem::Requirement
+  requirement: &70347429183120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914011460
+  version_requirements: *70347429183120
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70226914010720 !ruby/object:Gem::Requirement
+  requirement: &70347429181840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70226914010720
+  version_requirements: *70347429181840
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70226914010260 !ruby/object:Gem::Requirement
+  requirement: &70347429180860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,7 +131,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70226914010260
+  version_requirements: *70347429180860
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface
@@ -142,13 +142,14 @@ extensions: []
 extra_rdoc_files:
 - README.textile
 files:
+- spec/cobweb/cobweb_crawl_helper_spec.rb
 - spec/cobweb/cobweb_crawler_spec.rb
 - spec/cobweb/cobweb_job_spec.rb
 - spec/cobweb/cobweb_links_spec.rb
 - spec/cobweb/cobweb_spec.rb
 - spec/cobweb/content_link_parser_spec.rb
-- spec/cobweb/crawl_spec.rb
 - spec/cobweb/robots_spec.rb
+- spec/cobweb/site_test_spec.rb.tmp
 - spec/samples/robots.txt
 - spec/samples/sample_html_links.html
 - spec/samples/sample_server.rb
@@ -328,7 +329,9 @@ files:
 - lib/cobweb_process_job.rb
 - lib/cobweb_version.rb
 - lib/content_link_parser.rb
+- lib/crawl.rb
 - lib/crawl_job.rb
+- lib/crawl_object.rb
 - lib/encoding_safe_process_job.rb
 - lib/hash_util.rb
 - lib/redirect_error.rb