cobweb 1.0.18 → 1.0.19

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v1.0.18
+h1. Cobweb v1.0.19
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 !https://badge.fury.io/rb/cobweb.png!:http://badge.fury.io/rb/cobweb
@@ -95,7 +95,8 @@ Creates a new crawler object based on a base_url
   
     ** :follow_redirects              - transparently follows redirects and populates the :redirect_through key in the content hash (Default: true)
     ** :redirect_limit                - sets the limit to be used for concurrent redirects (Default: 10) 
-    ** :processing_queue              - specifies the processing queue for content to be sent to (Default: ContentProcessJob) 
+    ** :processing_queue              - specifies the processing queue for content to be sent to (Default: 'CobwebProcessJob' when using resque, 'CrawlProcessWorker' when using sidekiq) 
+    ** :crawl_finished_queue          - specifies the processing queue for statistics to be sent to after finishing crawling (Default: 'CobwebFinishedJob' when using resque, 'CrawlFinishedWorker' when using sidekiq)
     ** :debug                         - enables debug output (Default: false)
     ** :quiet                         - hides default output (Default: false)
     ** :cache                         - sets the ttl for caching pages, set to nil to disable caching (Default: 300)
@@ -110,9 +111,12 @@ Creates a new crawler object based on a base_url
     ** :obey_robots                   - boolean determining if robots.txt should be honoured. (default: false)
     ** :user_agent                    - user agent string to match in robots.txt (not sent as user_agent of requests yet) (default: cobweb)
     ** :crawl_limit_by_page           - sets the crawl counter to only use html page types when counting objects crawled
-    ** :valid_mime_types              - an array of mime types that takes wildcards (eg 'text/*') defaults to ['*/*']
+    ** :valid_mime_types              - an array of mime types that takes wildcards (eg 'text/*') defaults to @['*/*']@
     ** :direct_call_process_job       - boolean that specifies whether objects should be passed directly to a processing method or should be put onto a queue
     ** :raise_exceptions              - defaults to handling exceptions with debug output, setting this to true will raise exceptions in your app
+    ** :use_encoding_safe_process_job - Base64-encode the body when storing job in queue; set to true when you are expecting non-ASCII content (Default: false)
+    ** :proxy_addr                    - hostname of a proxy to use for crawling (e. g., 'myproxy.example.net', default: nil)
+    ** :proxy_port                    - port number of the proxy (default: nil)
     
     
 bc. crawler = Cobweb.new(:follow_redirects => false)

data/lib/cobweb.rb

@@ -9,6 +9,9 @@ Dir[File.dirname(__FILE__) + '/**/*.rb'].each do |file|
   require file
 end
 
+puts Gem::Specification.find_all_by_name("sidekiq", ">=3.0.0") 
+
+
 # Cobweb class is used to perform get and head requests.  You can use this on its own if you wish without the crawler
 class Cobweb
   
@@ -57,6 +60,8 @@ class Cobweb
     default_valid_mime_types_to                ["*/*"]
     default_raise_exceptions_to               false
     default_store_inbound_links_to            false
+    default_proxy_addr_to                     nil
+    default_proxy_port_to                     nil
 
   end
   
@@ -76,7 +81,7 @@ class Cobweb
     end
     
     request.merge!(@options)
-    @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{request[:crawl_id]}", :redis => Redis.new(request[:redis_options]))
+    @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{request[:crawl_id]}", :redis => RedisConnection.new(request[:redis_options]))
     @redis.set("original_base_url", base_url)
     @redis.hset "statistics", "queued_at", DateTime.now
     @redis.set("crawl-counter", 0)
@@ -130,11 +135,11 @@ class Cobweb
     
     # connect to redis
     if options.has_key? :crawl_id
-      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{options[:crawl_id]}", :redis => Redis.new(@options[:redis_options]))
+      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{options[:crawl_id]}", :redis => RedisConnection.new(@options[:redis_options]))
     else
-      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => Redis.new(@options[:redis_options]))
+      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => RedisConnection.new(@options[:redis_options]))
     end
-    full_redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => Redis.new(@options[:redis_options]))
+    full_redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => RedisConnection.new(@options[:redis_options]))
 
     content = {:base_url => url}
 
@@ -151,7 +156,7 @@ class Cobweb
       # retrieve data
       #unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." if @options[:debug]
-        @http = Net::HTTP.new(uri.host, uri.inferred_port)
+        @http = Net::HTTP.new(uri.host, uri.inferred_port, @options[:proxy_addr], @options[:proxy_port])
       #end
       if uri.scheme == "https"
         @http.use_ssl = true
@@ -309,9 +314,9 @@ class Cobweb
     
     # connect to redis
     if options.has_key? :crawl_id
-      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{options[:crawl_id]}", :redis => Redis.new(@options[:redis_options]))
+      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{options[:crawl_id]}", :redis => RedisConnection.new(@options[:redis_options]))
     else
-      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => Redis.new(@options[:redis_options]))
+      redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => RedisConnection.new(@options[:redis_options]))
     end
     
     content = {:base_url => url}
@@ -324,7 +329,7 @@ class Cobweb
       # retrieve data
       unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." unless @options[:quiet]
-        @http = Net::HTTP.new(uri.host, uri.inferred_port)
+        @http = Net::HTTP.new(uri.host, uri.inferred_port, @options[:proxy_addr], @options[:proxy_port])
       end
       if uri.scheme == "https"
         @http.use_ssl = true

data/lib/cobweb_crawler.rb

@@ -20,7 +20,7 @@ class CobwebCrawler
       @options[:crawl_id] = @crawl_id
     end
     
-    @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@crawl_id}", :redis => Redis.new(@options[:redis_options]))
+    @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@crawl_id}", :redis => RedisConnection.new(@options[:redis_options]))
     @options[:internal_urls] = [] if @options[:internal_urls].nil?
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
     @options[:seed_urls] = [] if @options[:seed_urls].nil?

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "1.0.18"
+    "1.0.19"
   end
 
 end

data/lib/crawl.rb

@@ -6,7 +6,7 @@ module CobwebModule
 
       setup_defaults
 
-      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@options[:crawl_id]}", :redis => Redis.new(@options[:redis_options]))
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@options[:crawl_id]}", :redis => RedisConnection.new(@options[:redis_options]))
       @stats = Stats.new(@options)
       @debug = @options[:debug]
       @first_to_finish = false
@@ -22,6 +22,15 @@ module CobwebModule
       @redis.sismember "queued", link
     end
 
+    def already_running?(link)
+      @redis.sismember "currently_running", link
+    end
+
+    def already_handled?(link)
+      already_crawled?(link) || already_queued?(link) || already_running?(link)
+    end
+
+
     # Returns true if the crawl count is within limits
     def within_crawl_limits?
       @options[:crawl_limit].nil? || crawl_counter < @options[:crawl_limit].to_i
@@ -50,16 +59,19 @@ module CobwebModule
     end
 
     def retrieve
-      unless @redis.sismember("currently_running", @options[:url])
-        @redis.sadd("currently_running", @options[:url])
-        unless already_crawled?
+
+      unless already_running? @options[:url]
+        unless already_crawled? @options[:url]
+          @redis.sadd("currently_running", @options[:url])
           if within_crawl_limits?
             @stats.update_status("Retrieving #{@options[:url]}...")
-            @content = Cobweb.new(@options).get(@options[:url], @options)
-            if @options[:url] == @redis.get("original_base_url")
-              @redis.set("crawled_base_url", @content[:base_url])
+            lock("update_queues") do
+              @content = Cobweb.new(@options).get(@options[:url], @options)
+              if @options[:url] == @redis.get("original_base_url")
+                @redis.set("crawled_base_url", @content[:base_url])
+              end
+              update_queues
             end
-            update_queues
 
             if content.permitted_type?
               ## update statistics
@@ -128,7 +140,7 @@ module CobwebModule
     end
 
     def update_queues
-      lock("update_queues") do
+      #lock("update_queues") do
         #@redis.incr "inprogress"
         # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
         @redis.srem "queued", @options[:url]
@@ -146,25 +158,27 @@ module CobwebModule
           increment_crawl_counter
         end
         decrement_queue_counter
-      end
+      #end
     end
 
     def to_be_processed?
-      (!finished? || within_process_limits?) && !@redis.sismember("enqueued", @options[:url])
+      !finished? && within_process_limits? && !@redis.sismember("queued", @options[:url])
     end
 
     def process(&block)
-      if @options[:crawl_limit_by_page]
-        if content.mime_type.match("text/html")
+      lock("process") do
+        if @options[:crawl_limit_by_page]
+          if content.mime_type.match("text/html")
+            increment_process_counter
+          end
+        else
           increment_process_counter
         end
-      else
-        increment_process_counter
-      end
-      @redis.sadd "enqueued", @options[:url]
+        #@redis.sadd "queued", @options[:url]
 
-      yield if block_given?
-      @redis.incr("crawl_job_enqueued_count")
+        yield if block_given?
+        @redis.incr("crawl_job_enqueued_count")
+      end
     end
 
     def finished_processing
@@ -173,20 +187,33 @@ module CobwebModule
 
     def finished?
       print_counters
+      debug_puts @stats.get_status
+      if @stats.get_status == CobwebCrawlHelper::FINISHED
+        debug_puts "Already Finished!"
+      end  
       # if there's nothing left queued or the crawled limit has been reached and we're not still processing something
       if @options[:crawl_limit].nil? || @options[:crawl_limit] == 0
         if queue_counter == 0 && @redis.smembers("currently_running").empty?
-          finished
+          debug_puts "queue_counter is 0 and currently_running is empty so we're done"
+          #finished
           return true
         end
-      elsif (queue_counter == 0 && @redis.smembers("currently_running").empty?) || process_counter >= @options[:crawl_limit].to_i
-        finished
+      elsif (queue_counter == 0 || process_counter >= @options[:crawl_limit].to_i) && @redis.smembers("currently_running").empty?
+        #finished
+        debug_puts "queue_counter: #{queue_counter}, @redis.smembers(\"currently_running\").empty?: #{@redis.smembers("currently_running").empty?}, process_counter: #{process_counter}, @options[:crawl_limit].to_i: #{@options[:crawl_limit].to_i}"
         return true
       end
       false
     end
 
-    def finished
+    def finish
+      debug_puts ""
+      debug_puts "========================================================================"
+      debug_puts "finished crawl on #{@options[:url]}"
+      print_counters
+      debug_puts "========================================================================"
+      debug_puts ""
+
       set_first_to_finish
       @stats.end_crawl(@options)
     end
@@ -223,22 +250,22 @@ module CobwebModule
     end
 
     def lock(key, &block)
-      debug_puts "REQUESTING LOCK [#{key}]"
+      #debug_puts "REQUESTING LOCK [#{key}]"
       set_nx = @redis.setnx("#{key}_lock", "locked")
-      debug_puts "LOCK:#{key}:#{set_nx}"
+      #debug_puts "LOCK:#{key}:#{set_nx}"
       while !set_nx
-        debug_puts "===== WAITING FOR LOCK [#{key}] ====="
+        #debug_puts "===== WAITING FOR LOCK [#{key}] ====="
         sleep 0.01
         set_nx = @redis.setnx("#{key}_lock", "locked")
       end
 
-      debug_puts "RECEIVED LOCK [#{key}]"
+      #debug_puts "RECEIVED LOCK [#{key}]"
       @redis.expire("#{key}_lock", 10)
       begin
         result = yield
       ensure
         @redis.del("#{key}_lock")
-        debug_puts "LOCK RELEASED [#{key}]"
+        #debug_puts "LOCK RELEASED [#{key}]"
       end
       result
     end

data/lib/crawl_helper.rb

@@ -15,7 +15,7 @@ class CrawlHelper
     content_request[:valid_mime_types] = ["*/*"] unless content_request.has_key? :valid_mime_types
     content_request[:queue_system] = content_request[:queue_system].to_sym
     
-    @redis = NamespacedRedis.new(content_request[:redis_options], "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
+    @redis = NamespacedRedisConnection.new(content_request[:redis_options], "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
     @stats = Stats.new(content_request)
     
     @debug = content_request[:debug]
@@ -74,7 +74,7 @@ class CrawlHelper
 
             #if the enqueue counter has been requested update that
             if content_request.has_key? :enqueue_counter_key
-              enqueue_redis = NamespacedRedis.new(content_request[:redis_options], content_request[:enqueue_counter_namespace].to_s)
+              enqueue_redis = NamespacedRedisConnection.new(content_request[:redis_options], content_request[:enqueue_counter_namespace].to_s)
               current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
               enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
             end

data/lib/crawl_job.rb

@@ -23,12 +23,14 @@ class CrawlJob
         # extract links from content and process them if we are still within queue limits (block will not run if we are outwith limits)
         @crawl.process_links do |link|
 
-          # enqueue the links to resque
-          @crawl.debug_puts "ENQUEUED LINK: #{link}" 
-          enqueue_content(content_request, link) 
+          if @crawl.within_crawl_limits?
+            # enqueue the links to resque
+            @crawl.debug_puts "ENQUEUED LINK: #{link}" 
+            enqueue_content(content_request, link)
+          end
 
         end
-        
+    
         if @crawl.to_be_processed?
           
           @crawl.process do
@@ -39,7 +41,7 @@ class CrawlJob
 
             #if the enqueue counter has been requested update that
             if content_request.has_key?(:enqueue_counter_key)
-              enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => Redis.new(content_request[:redis_options]))
+              enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => RedisConnection.new(content_request[:redis_options]))
               current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
               enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
             end
@@ -60,8 +62,7 @@ class CrawlJob
 
       # test queue and crawl sizes to see if we have completed the crawl
       @crawl.debug_puts "finished? #{@crawl.finished?}"
-      @crawl.debug_puts "first_to_finish? #{@crawl.first_to_finish?}" if @crawl.finished?
-      if @crawl.finished? && @crawl.first_to_finish?
+      if @crawl.finished?
         @crawl.debug_puts "Calling crawl_job finished"
         finished(content_request)
       end
@@ -75,7 +76,9 @@ class CrawlJob
     additional_stats[:redis_options] = content_request[:redis_options] unless content_request[:redis_options] == {}
     additional_stats[:source_id] = content_request[:source_id] unless content_request[:source_id].nil?
     
-    @crawl.debug_puts "increment crawl_finished_enqueued_count"
+    @crawl.finish
+
+    @crawl.debug_puts "increment crawl_finished_enqueued_count from #{@crawl.redis.get("crawl_finished_enqueued_count")}"
     @crawl.redis.incr("crawl_finished_enqueued_count")
     Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @crawl.statistics.merge(additional_stats))
   end

data/lib/crawl_worker.rb

@@ -16,6 +16,7 @@ class CrawlWorker
   sidekiq_options :queue => "crawl_worker", :retry => false if SIDEKIQ_INSTALLED
   
   def perform(content_request)
+    puts "Performing for #{content_request["url"]}"
     # setup the crawl class to manage the crawl of this object
     @crawl = CobwebModule::Crawl.new(content_request)
     
@@ -25,12 +26,17 @@ class CrawlWorker
       # if the crawled object is an object type we are interested
       if @crawl.content.permitted_type?
         
-        # extract links from content and process them if we are still within queue limits (block will not run if we are outwith limits)
-        @crawl.process_links do |link|
+        @crawl.lock("queue_links") do
+          # extract links from content and process them if we are still within queue limits (block will not run if we are outwith limits)
+          @crawl.process_links do |link|
 
-          @crawl.debug_puts "ENQUEUED LINK: #{link}" 
-          enqueue_content(content_request, link) 
+            if @crawl.within_crawl_limits? && !@crawl.already_handled?(link)
+              # enqueue the links to sidekiq
+              @crawl.debug_puts "QUEUED LINK: #{link}" 
+              enqueue_content(content_request, link)
+            end
 
+          end
         end
         
         if @crawl.to_be_processed?
@@ -38,12 +44,12 @@ class CrawlWorker
           @crawl.process do
 
             # enqueue to processing queue
-            @crawl.debug_puts "ENQUEUED [#{@crawl.redis.get("crawl_job_enqueued_count")}] URL: #{@crawl.content.url}"
+            @crawl.debug_puts "SENT FOR PROCESSING [#{@crawl.redis.get("crawl_job_enqueued_count")}] URL: #{@crawl.content.url}"
             send_to_processing_queue(@crawl.content.to_hash, content_request)
 
             #if the enqueue counter has been requested update that
             if content_request.has_key?(:enqueue_counter_key)
-              enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => Redis.new(content_request[:redis_options]))
+              enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => RedisConnection.new(content_request[:redis_options]))
               current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
               enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
             end
@@ -64,8 +70,7 @@ class CrawlWorker
 
       # test queue and crawl sizes to see if we have completed the crawl
       @crawl.debug_puts "finished? #{@crawl.finished?}"
-      @crawl.debug_puts "first_to_finish? #{@crawl.first_to_finish?}" if @crawl.finished?
-      if @crawl.finished? && @crawl.first_to_finish?
+      if @crawl.finished?
         @crawl.debug_puts "Calling crawl_job finished"
         finished(content_request)
       end
@@ -84,7 +89,7 @@ class CrawlWorker
     additional_stats[:redis_options] = content_request[:redis_options] unless content_request[:redis_options] == {}
     additional_stats[:source_id] = content_request[:source_id] unless content_request[:source_id].nil?
 
-    @crawl.finished
+    @crawl.finish
 
     @crawl.debug_puts "increment crawl_finished_enqueued_count"
     @crawl.redis.incr("crawl_finished_enqueued_count")

data/lib/redis_connection.rb

@@ -0,0 +1,23 @@
+class RedisConnection
+
+  @@redis_connections = {}
+
+  def initialize(options={})
+    key = options.keys.sort.map{|k| "#{k}:#{options[k]}"}.join(",")
+    unless @@redis_connections.has_key?(key)
+      @@redis_connections[key] = Redis.new(options)
+    end
+    @current_connection = @@redis_connections[key]
+    @current_connection
+  end
+
+  def method_missing(m, *args, &block)
+    if @current_connection.respond_to?(m)
+      @current_connection.send(m, *args)
+    else
+      super
+    end
+  end
+
+
+end

data/lib/server.rb

@@ -12,14 +12,14 @@ class Server < Sinatra::Base
   
   # Sinatra Dashboard
   get '/' do
-    @full_redis = Redis.new(redis_options)
+    @full_redis = RedisConnection.new(redis_options)
     @colors = ["#00366f", "#006ba0", "#3F0BDB", "#396CB3"]
     
     @crawls = []
     @full_redis.smembers("cobweb_crawls").each do |crawl_id|
       version = cobweb_version(crawl_id)
       if version == Cobweb.version
-        redis = Redis::Namespace.new("cobweb-#{version}-#{crawl_id}", :redis => Redis.new(redis_options))
+        redis = Redis::Namespace.new("cobweb-#{version}-#{crawl_id}", :redis => RedisConnection.new(redis_options))
         stats = HashUtil.deep_symbolize_keys({
           :cobweb_version => version,
           :crawl_details => redis.hgetall("crawl_details"),
@@ -38,7 +38,7 @@ class Server < Sinatra::Base
   get '/statistics/:crawl_id' do
     
     version = cobweb_version(params[:crawl_id])
-    redis = Redis::Namespace.new("cobweb-#{version}-#{params[:crawl_id]}", :redis => Redis.new(redis_options))
+    redis = Redis::Namespace.new("cobweb-#{version}-#{params[:crawl_id]}", :redis => RedisConnection.new(redis_options))
     
     @statistics = HashUtil.deep_symbolize_keys(redis.hgetall("statistics"))
     if @statistics[:status_counts].nil?
@@ -71,7 +71,7 @@ class Server < Sinatra::Base
   end
   
   def cobweb_version(crawl_id)
-    redis = Redis.new(redis_options)
+    redis = RedisConnection.new(redis_options)
     key = redis.keys("cobweb-*-#{crawl_id}:queued").first
     
     key =~ /cobweb-(.*?)-(.*?):queued/

data/lib/sidekiq/cobweb_helper.rb

@@ -1,9 +1,10 @@
-if Gem::Specification.find_all_by_name("sidekiq", ">=3.0.0").count > 1
+
+if Gem::Specification.find_all_by_name("sidekiq", ">=1.0.0").count >= 1
   SIDEKIQ_INSTALLED = true
   require 'sidekiq'
 else
   SIDEKIQ_INSTALLED = false
-  puts "can't find sidekiq gem"
+  puts "sidekiq gem not installed, skipping crawl_worker specs"
 end
 
 module Sidekiq

data/spec/cobweb/cobweb_spec.rb

@@ -31,6 +31,8 @@ describe Cobweb do
     options[:timeout].should == 10
     options[:redis_options].should == {}
     options[:internal_urls].should == []
+    options[:proxy_addr].should be_nil
+    options[:proxy_port].should be_nil
     
   end
   
@@ -52,15 +54,15 @@ describe Cobweb do
         @cobweb.get(@base_url)[:url].should == @base_url
       end
       it "should return correct content-type" do
-        @mock_http_response.stub!(:content_type).and_return("image/jpeg")
+        @mock_http_response.stub(:content_type).and_return("image/jpeg")
         @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
       end
       it "should return correct status-code" do
-        @mock_http_response.stub!(:code).and_return(404)
+        @mock_http_response.stub(:code).and_return(404)
         @cobweb.get(@base_url)[:status_code].should == 404
       end
       it "should return correct status-code" do
-        @mock_http_response.stub!(:code).and_return(404)
+        @mock_http_response.stub(:code).and_return(404)
         @cobweb.get(@base_url)[:status_code].should == 404
       end
       it "should return correct character_set" do
@@ -75,7 +77,7 @@ describe Cobweb do
       it "should return correct location" do
         @cobweb.get(@base_url)[:location].should == nil
 
-        @mock_http_response.stub!(:[]).with("location").and_return("http://google.com/")
+        @mock_http_response.stub(:[]).with("location").and_return("http://google.com/")
         @cobweb.get(@base_url)[:location].should == "http://google.com/"
       end
       it "should return correct headers" do
@@ -135,17 +137,17 @@ describe Cobweb do
           @cobweb.get(@base_url)[:url].should == @base_url
         end
         it "should return correct content-type" do
-          @mock_http_response.stub!(:content_type).and_return("image/jpeg")
+          @mock_http_response.stub(:content_type).and_return("image/jpeg")
           @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
           @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
         end
         it "should return correct status-code" do
-          @mock_http_response.stub!(:code).and_return(404)
+          @mock_http_response.stub(:code).and_return(404)
           @cobweb.get(@base_url)[:status_code].should == 404
           @cobweb.get(@base_url)[:status_code].should == 404
         end
         it "should return correct status-code" do
-          @mock_http_response.stub!(:code).and_return(404)
+          @mock_http_response.stub(:code).and_return(404)
           @cobweb.get(@base_url)[:status_code].should == 404
           @cobweb.get(@base_url)[:status_code].should == 404
         end
@@ -177,26 +179,34 @@ describe Cobweb do
     end
     describe "location setting" do
       it "Get should strip fragments" do
-        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, nil, nil)
         Net::HTTP::Get.should_receive(:new).with("/", @default_options)
         @cobweb.get("http://www.google.com/#ignore")
       end
       it "head should strip fragments" do
-        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, nil, nil)
         Net::HTTP::Head.should_receive(:new).with("/", {}).and_return(@mock_http_request)
         @cobweb.head("http://www.google.com/#ignore")
       end
       it "get should not strip path" do
-        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, nil, nil)
         Net::HTTP::Get.should_receive(:new).with("/path/to/stuff", @default_options)
         @cobweb.get("http://www.google.com/path/to/stuff#ignore")
       end
       it "get should not strip query string" do
-        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, nil, nil)
         Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string", @default_options)
         @cobweb.get("http://www.google.com/path/to/stuff?query_string#ignore")
       end
     end
+    describe "with proxy" do
+      it "provides proxy parameters to Net::HTTP" do
+        cobweb = Cobweb.new proxy_addr: 'proxy.example.com', proxy_port: 1234
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, "proxy.example.com", 1234)
+
+        cobweb.get("http://www.google.com/")
+      end
+    end
 
   end  
 end 

data/spec/cobweb/crawl_job_spec.rb

@@ -9,7 +9,7 @@ describe CrawlJob, :local_only => true, :disabled => true do
 
     @existing_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
     if Resque.workers.count > 0 && @existing_processes.empty?
-      raise "Ghost workers present in resque, please clear before running specs"
+      raise "Ghost workers present in resque, please clear before running specs (Resque::Worker.all.first.prune_dead_workers)"
     elsif Resque.workers.count == 0 && !@existing_processes.empty?
       raise "Ghost worker processes present (#{@existing_processes.join(',')})"
     elsif Resque.workers.count > 0 && !@existing_processes.empty?
@@ -23,25 +23,23 @@ describe CrawlJob, :local_only => true, :disabled => true do
     io = IO.popen("nohup rake resque:workers INTERVAL=1 PIDFILE=./tmp/pids/resque.pid COUNT=#{RESQUE_WORKER_COUNT} QUEUE=cobweb_crawl_job > log/output.log &")
 
     counter = 0
-    print "Starting Resque Processes"
     until counter > 10 || workers_processes_started?
-      print "."
+      print "\rStarting Resque Processes... #{10-counter} "
       counter += 1
-      sleep 0.5
+      sleep 1
     end
     puts ""
 
 
     counter = 0
-    print "Waiting for Resque Workers"
-    until counter > 50 || workers_running?
-      print "."
+    until counter > 30 || workers_running?
+      print "\rWaiting for Resque Workers... #{30-counter} "
       counter += 1
-      sleep 0.5
+      sleep 1
     end
     puts ""
 
-    if Resque.workers.count == RESQUE_WORKER_COUNT
+    if workers_running?
       puts "Workers Running."
     else
       raise "Workers didn't appear, please check environment"
@@ -62,10 +60,10 @@ describe CrawlJob, :local_only => true, :disabled => true do
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
         :crawl_limit => nil,
         :quiet => false,
-        :debug => false,
+        :debug => true,
         :cache => nil
       }
-      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", :redis => Redis.new)
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", :redis => RedisConnection.new)
       @cobweb = Cobweb.new @request
     end
     it "should not crawl anything if nothing has started" do
@@ -95,10 +93,10 @@ describe CrawlJob, :local_only => true, :disabled => true do
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
         :crawl_limit => nil,
         :quiet => false,
-        :debug => false,
+        :debug => true,
         :cache => nil
       }
-      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", :redis => Redis.new)
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", :redis => RedisConnection.new)
 
       @cobweb = Cobweb.new @request
     end
@@ -124,11 +122,11 @@ describe CrawlJob, :local_only => true, :disabled => true do
       @request = {
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
         :quiet => false,
-        :debug => false,
+        :debug => true,
         :cache => nil,
         :valid_mime_types => ["text/html"]
       }
-      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", :redis => Redis.new)
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", :redis => RedisConnection.new)
       @cobweb = Cobweb.new @request
     end
 
@@ -150,10 +148,10 @@ describe CrawlJob, :local_only => true, :disabled => true do
       @request = {
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
         :quiet => false,
-        :debug => false,
+        :debug => true,
         :cache => nil
       }
-      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", :redis => Redis.new)
+      @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@request[:crawl_id]}", :redis => RedisConnection.new)
     end
 
     # describe "crawling http://yepadeperrors.wordpress.com/ with limit of 20" do
@@ -226,6 +224,7 @@ describe CrawlJob, :local_only => true, :disabled => true do
         @redis.get("crawl_job_enqueued_count").to_i.should_not == @base_page_count
       end
       it "should notify of crawl finished once" do
+        @redis.get("crawl_finished_enqueued_count").to_i.should == 0
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
@@ -280,11 +279,13 @@ end
 
 def wait_for_crawl_finished(crawl_id, timeout=20)
   @counter = 0
+  @timeout = timeout unless @timeout
   start_time = Time.now
   while(running?(crawl_id) && Time.now < start_time + timeout) do
     sleep 1
   end
-  if Time.now > start_time + timeout
+  if Time.now > start_time + @timeout
+    @timeout = 5
     raise "End of crawl not detected"
   end
 end
@@ -296,7 +297,7 @@ def workers_processes_started?
 end
 
 def workers_running?
-  Resque.workers.count > 0
+  Resque.workers.count == RESQUE_WORKER_COUNT
 end
 
 def running?(crawl_id)

data/spec/cobweb/crawl_worker_spec.rb

@@ -8,8 +8,8 @@ describe CrawlWorker, :local_only => true do
     if SIDEKIQ_INSTALLED    
       #store all existing resque process ids so we don't kill them afterwards
       @existing_processes = `ps aux | grep sidekiq | grep -v grep | awk '{print $2}'`.split("\n")
-      puts @existing_processes
-      @existing_processes.should be_empty
+
+      raise "Sidekiq is already running, please stop before running specs." if @existing_processes.count > 0
     
       # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES
       puts "Starting Workers... Please Wait..."
@@ -34,7 +34,7 @@ describe CrawlWorker, :local_only => true do
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
         :crawl_limit => nil,
         :quiet => false,
-        :debug => false,
+        :debug => true,
         :cache => nil,
         :queue_system => :sidekiq
       }
@@ -60,6 +60,7 @@ describe CrawlWorker, :local_only => true do
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
         :quiet => true,
         :cache => nil,
+        :debug => true,
         :queue_system => :sidekiq,
         :valid_mime_types => ["text/html"]
       }
@@ -87,6 +88,7 @@ describe CrawlWorker, :local_only => true do
       @request = {
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
         :quiet => true,
+        :debug => true,
         :queue_system => :sidekiq,
         :cache => nil
       }
@@ -136,7 +138,6 @@ describe CrawlWorker, :local_only => true do
           wait_for_crawl_finished crawl[:crawl_id]
         
           mime_types = CrawlProcessWorker.queue_items(0, 200).map{|job| JSON.parse(job)["args"][0]["mime_type"]}
-          ap mime_types
           mime_types.select{|m| m=="text/html"}.count.should == 5
         end
       end
@@ -186,11 +187,11 @@ describe CrawlWorker, :local_only => true do
         wait_for_crawl_finished crawl[:crawl_id]
         CrawlFinishedWorker.queue_size.should == 1
       end      
-      it "should not crawl 100 pages" do
+      it "should not crawl more than 100 pages" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        CrawlProcessWorker.queue_size.should_not == 100
+        CrawlProcessWorker.queue_size.should_not > 100
       end      
     end    
   end
@@ -244,7 +245,7 @@ def clear_sidekiq_queues
       conn.srem("queues", queue_name)
     end
   end
-  sleep 2
+  sleep 5
   
   CrawlProcessWorker.queue_size.should == 0
   CrawlFinishedWorker.queue_size.should == 0

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.18
+  version: 1.0.19
+  prerelease: 
 platform: ruby
 authors:
 - Stewart McKee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-17 00:00:00.000000000 Z
+date: 2013-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -27,6 +30,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -41,6 +46,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -48,6 +54,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -55,6 +62,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,6 +70,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -69,6 +78,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -76,6 +86,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -83,6 +94,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -90,6 +102,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -97,6 +110,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: thin
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -104,6 +118,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -111,6 +126,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: haml
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -118,6 +134,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -125,6 +142,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -132,6 +150,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -139,6 +158,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -146,6 +166,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -153,6 +174,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -160,6 +182,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -376,6 +399,7 @@ files:
 - lib/export_command.rb
 - lib/hash_util.rb
 - lib/redirect_error.rb
+- lib/redis_connection.rb
 - lib/report_command.rb
 - lib/robots.rb
 - lib/server.rb
@@ -540,26 +564,27 @@ files:
 homepage: http://github.com/stewartmckee/cobweb
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.25
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites faster than multi-threaded crawlers.  It is also a standalone
   crawler that has a sophisticated statistics monitoring interface to monitor the

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MTg3ODFiMWE1MmZlYWFjYzZiZjIzZjQ1NmFjZmJmMWU1MDVjZTc5Mg==
-  data.tar.gz: !binary |-
-    ZGU4NmFiYTJlNmZlODRiMjRmNTkzZjMwOWQyMzEyZjU4OGQzMWUxMw==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    ZDVmN2MwYzBiMjQ1N2E2YjBmYmM0ZTk5ZWJjMGVkN2VmMDM4ODhkNTQ0OTIx
-    ZTg4YzMzMWE0OTY2ZjgyNWRiNzZlZjgyZDlkM2Y4MTQ2OTVmZTg5Zjc1NTA1
-    MTZhYzc2ZmYwNmM2ODRlMmViODljMGFjODYwNTY5OThlNjY2M2Y=
-  data.tar.gz: !binary |-
-    M2M2YzU4ZTE5YzkxMWVmNmJiNTQ5OWFhNDExZGUwNzkxMGEzY2IyYTFmYTJl
-    YTE0OWI2ZmZhN2I0ZjA2YjU4NWFmNmUwMjY5ZDM4YWQ3ZmJkZmViNzRlNWMw
-    ZWMzNjIwNDkxNDk0NmMxOTE3NzljMGQ5MjlmYzgyODc3ZWQ2ZTY=