cobweb 1.0.6 → 1.0.8

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v1.0.6
+h1. Cobweb v1.0.8
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 
@@ -152,6 +152,10 @@ bc. statistics = CobwebCrawler.new(:cache => 600).crawl("http://www.pepsico.com"
 end
 puts "Finished Crawl with #{statistics[:page_count]} pages and #{statistics[:asset_count]} assets."
 
+There are some specific options for CobwebCrawler in addition to the normal cobweb options
+
+  * thread_count - specifies the number of threads used by the crawler, defaults to 1
+
 h3. CobwebCrawlHelper
 
 The CobwebCrawlHelper class is a helper class to assist in getting information about a crawl and to perform functions against the crawl

data/lib/cobweb.rb

@@ -127,10 +127,10 @@ class Cobweb
       content = HashUtil.deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
     else
       # retrieve data
-      unless @http && @http.address == uri.host && @http.port == uri.inferred_port
+      #unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." if @options[:debug]
         @http = Net::HTTP.new(uri.host, uri.inferred_port)
-      end
+      #end
       if uri.scheme == "https"
         @http.use_ssl = true
         @http.verify_mode = OpenSSL::SSL::VERIFY_NONE

data/lib/cobweb_crawler.rb

@@ -39,6 +39,7 @@ class CobwebCrawler
   # Initiates a crawl starting at the base_url and applying the options supplied. Can also take a block that is executed and passed content hash and statistic hash'
   def crawl(base_url, crawl_options = {}, &block)
     @options[:base_url] = base_url unless @options.has_key? :base_url
+    @options[:thread_count] = 1 unless @options.has_key? :thread_count
     
     @options[:internal_urls] << base_url if @options[:internal_urls].empty?
     @redis.sadd("internal_urls", base_url) if @options[:internal_urls].empty?
@@ -46,87 +47,110 @@ class CobwebCrawler
     @crawl_options = crawl_options
     
     @redis.sadd("queued", base_url) unless base_url.nil? || @redis.sismember("crawled", base_url) || @redis.sismember("queued", base_url)
-    crawl_counter = @redis.scard("crawled").to_i
-    queue_counter = @redis.scard("queued").to_i
+    @crawl_counter = @redis.scard("crawled").to_i
+    @queue_counter = @redis.scard("queued").to_i
 
+    @threads = []
     begin
       @stats.start_crawl(@options)
-      while queue_counter>0 && (@options[:crawl_limit].to_i == 0 || @options[:crawl_limit].to_i > crawl_counter)      
-        thread = Thread.new do
-        
+      
+      @threads << Thread.new do
+        Thread.abort_on_exception = true
+        spawn_thread(&block)
+      end
 
-          url = @redis.spop "queued"
-          queue_counter = 0 if url.nil?
+      sleep 5
+      while running_thread_count > 0
+        if @queue_counter > 0
+          (@options[:thread_count]-running_thread_count).times.each do
+            @threads << Thread.new do
+              Thread.abort_on_exception = true
+              spawn_thread(&block)
+            end
+          end
+        end
+        sleep 1
+      end
+      
+    ensure
+      @stats.end_crawl(@options)
+    end
+    @stats
+  end
 
-          @options[:url] = url
-          unless @redis.sismember("crawled", url.to_s)
-            begin
-              @stats.update_status("Requesting #{url}...")
-              content = @cobweb.get(url) unless url.nil?
-              if content.nil?
-                queue_counter = queue_counter - 1 #@redis.scard("queued").to_i
-              else
-                @stats.update_status("Processing #{url}...")
+  def spawn_thread(&block)
+      while @queue_counter>0 && (@options[:crawl_limit].to_i == 0 || @options[:crawl_limit].to_i > @crawl_counter)
+        url = @redis.spop "queued"
+      @queue_counter = 0 if url.nil?
 
-                @redis.sadd "crawled", url.to_s
-                @redis.incr "crawl-counter" 
-              
-                internal_links = ContentLinkParser.new(url, content[:body]).all_links(:valid_schemes => [:http, :https])
+      @options[:url] = url
+      unless @redis.sismember("crawled", url.to_s)
+        begin
+          @stats.update_status("Requesting #{url}...")
+          content = @cobweb.get(url) unless url.nil?
+          if content.nil?
+            @queue_counter = @queue_counter - 1 #@redis.scard("queued").to_i
+          else
+            @stats.update_status("Processing #{url}...")
+
+            @redis.sadd "crawled", url.to_s
+            @redis.incr "crawl-counter" 
+          
+            internal_links = ContentLinkParser.new(url, content[:body]).all_links(:valid_schemes => [:http, :https])
 
-                # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
-                cobweb_links = CobwebLinks.new(@options)
+            # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
+            cobweb_links = CobwebLinks.new(@options)
 
-                internal_links = internal_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s))}
+            internal_links = internal_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s))}
 
-                all_internal_links = internal_links
-                
-                # reject the link if we've crawled it or queued it
-                internal_links.reject!{|link| @redis.sismember("crawled", link)}
-                internal_links.reject!{|link| @redis.sismember("queued", link)}
-                internal_links.reject!{|link| link.nil? || link.empty?}
-              
-                internal_links.each do |link|
-                  puts "Added #{link.to_s} to queue" if @debug
-                  @redis.sadd "queued", link unless link.nil?
-                  children = @redis.hget("navigation", url)
-                  children = [] if children.nil?
-                  children << link
-                  @redis.hset "navigation", url, children
-                  queue_counter += 1
-                end
+            all_internal_links = internal_links
+            
+            # reject the link if we've crawled it or queued it
+            internal_links.reject!{|link| @redis.sismember("crawled", link)}
+            internal_links.reject!{|link| @redis.sismember("queued", link)}
+            internal_links.reject!{|link| link.nil? || link.empty?}
+          
+            internal_links.each do |link|
+              puts "Added #{link.to_s} to queue" if @debug
+              @redis.sadd "queued", link unless link.nil?
+              children = @redis.hget("navigation", url)
+              children = [] if children.nil?
+              children << link
+              @redis.hset "navigation", url, children
+              @queue_counter += 1
+            end
 
-                if @options[:store_refered_url]
-                  all_internal_links.each do |link|
-                    @redis.sadd("inbound_links_#{Digest::MD5.hexdigest(link)}", url)
-                  end
-                end
-              
-                crawl_counter = @redis.scard("crawled").to_i
-                queue_counter = @redis.scard("queued").to_i
-              
-                @stats.update_statistics(content, crawl_counter, queue_counter)
-                @stats.update_status("Completed #{url}.")
-                yield content, @stats.get_statistics if block_given?
+            if @options[:store_refered_url]
+              all_internal_links.each do |link|
+                @redis.sadd("inbound_links_#{Digest::MD5.hexdigest(link)}", url)
               end
-            rescue => e
-              puts "Error loading #{url}: #{e}"
-              #puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
-              #ap e
-              #ap e.backtrace
-            ensure
-              crawl_counter = @redis.scard("crawled").to_i
-              queue_counter = @redis.scard("queued").to_i
             end
-          else
-            puts "Already crawled #{@options[:url]}" if @debug
+          
+            @crawl_counter = @redis.scard("crawled").to_i
+            @queue_counter = @redis.scard("queued").to_i
+          
+            @stats.update_statistics(content, @crawl_counter, @queue_counter)
+            @stats.update_status("Completed #{url}.")
+            yield content, @stats.get_statistics if block_given?
           end
+        rescue => e
+          puts "Error loading #{url}: #{e}"
+          #puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
+          #ap e
+          #ap e.backtrace
+        ensure
+          @crawl_counter = @redis.scard("crawled").to_i
+          @queue_counter = @redis.scard("queued").to_i
         end
-        thread.join
+      else
+        puts "Already crawled #{@options[:url]}" if @debug
       end
-    ensure
-      @stats.end_crawl(@options)
     end
-    @stats
+    Thread.exit
+  end
+
+  def running_thread_count
+    @threads.map{|t| t.status}.select{|status| status=="run" || status == "sleep"}.count
   end
   
 end

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "1.0.6"
+    "1.0.8"
   end
 
 end

data/lib/stats.rb

@@ -8,6 +8,7 @@ class Stats
   def initialize(options)
     options[:redis_options] = {} unless options.has_key? :redis_options
     @full_redis = Redis.new(options[:redis_options])
+    @lock = Mutex.new
     @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{options[:crawl_id]}", :redis => @full_redis)
   end
   
@@ -44,105 +45,106 @@ class Stats
   # Returns statistics hash.  update_statistics takes the content hash, extracts statistics from it and updates redis with the data.  
   def update_statistics(content, crawl_counter=@redis.scard("crawled").to_i, queue_counter=@redis.scard("queued").to_i)
     
-    @statistics = get_statistics
-    
-    if @statistics.has_key? :average_response_time
-      @statistics[:average_response_time] = (((@redis.hget("statistics", "average_response_time").to_f*crawl_counter) + content[:response_time].to_f) / (crawl_counter + 1))
-    else
-      @statistics[:average_response_time] = content[:response_time].to_f
-    end
-    @statistics[:maximum_response_time] = content[:response_time].to_f if @statistics[:maximum_response_time].nil? or content[:response_time].to_f > @statistics[:maximum_response_time].to_f
-    @statistics[:minimum_response_time] = content[:response_time].to_f if @statistics[:minimum_response_time].nil? or content[:response_time].to_f < @statistics[:minimum_response_time].to_f
-    if @statistics.has_key? :average_length
-      @statistics[:average_length] = (((@redis.hget("statistics", "average_length").to_i*crawl_counter) + content[:length].to_i) / (crawl_counter + 1))
-    else
-      @statistics[:average_length] = content[:length].to_i
-    end
-    @statistics[:maximum_length] = content[:length].to_i if @redis.hget("statistics", "maximum_length").nil? or content[:length].to_i > @statistics[:maximum_length].to_i
-    @statistics[:minimum_length] = content[:length].to_i if @redis.hget("statistics", "minimum_length").nil? or content[:length].to_i < @statistics[:minimum_length].to_i
-    
-    if content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
-      @statistics[:page_count] = @statistics[:page_count].to_i + 1
-      @statistics[:page_size] = @statistics[:page_size].to_i + content[:length].to_i
-      increment_time_stat("pages_count")
-    else
-      @statistics[:asset_count] = @statistics[:asset_count].to_i + 1
-      @statistics[:asset_size] = @statistics[:asset_size].to_i + content[:length].to_i
-      increment_time_stat("assets_count")
-    end
-    
-    total_redirects = @statistics[:total_redirects].to_i
-    @statistics[:total_redirects] = 0 if total_redirects.nil?
-    @statistics[:total_redirects] = total_redirects += content[:redirect_through].count unless content[:redirect_through].nil?
+    @lock.synchronize {
+      @statistics = get_statistics
+      
+      if @statistics.has_key? :average_response_time
+        @statistics[:average_response_time] = (((@redis.hget("statistics", "average_response_time").to_f*crawl_counter) + content[:response_time].to_f) / (crawl_counter + 1))
+      else
+        @statistics[:average_response_time] = content[:response_time].to_f
+      end
+      @statistics[:maximum_response_time] = content[:response_time].to_f if @statistics[:maximum_response_time].nil? or content[:response_time].to_f > @statistics[:maximum_response_time].to_f
+      @statistics[:minimum_response_time] = content[:response_time].to_f if @statistics[:minimum_response_time].nil? or content[:response_time].to_f < @statistics[:minimum_response_time].to_f
+      if @statistics.has_key? :average_length
+        @statistics[:average_length] = (((@redis.hget("statistics", "average_length").to_i*crawl_counter) + content[:length].to_i) / (crawl_counter + 1))
+      else
+        @statistics[:average_length] = content[:length].to_i
+      end
+      @statistics[:maximum_length] = content[:length].to_i if @redis.hget("statistics", "maximum_length").nil? or content[:length].to_i > @statistics[:maximum_length].to_i
+      @statistics[:minimum_length] = content[:length].to_i if @redis.hget("statistics", "minimum_length").nil? or content[:length].to_i < @statistics[:minimum_length].to_i
+      
+      if content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
+        @statistics[:page_count] = @statistics[:page_count].to_i + 1
+        @statistics[:page_size] = @statistics[:page_size].to_i + content[:length].to_i
+        increment_time_stat("pages_count")
+      else
+        @statistics[:asset_count] = @statistics[:asset_count].to_i + 1
+        @statistics[:asset_size] = @statistics[:asset_size].to_i + content[:length].to_i
+        increment_time_stat("assets_count")
+      end
+      
+      total_redirects = @statistics[:total_redirects].to_i
+      @statistics[:total_redirects] = 0 if total_redirects.nil?
+      @statistics[:total_redirects] = total_redirects += content[:redirect_through].count unless content[:redirect_through].nil?
 
-    @statistics[:crawl_counter] = crawl_counter
-    @statistics[:queue_counter] = queue_counter
-    
-    total_length = @statistics[:total_length].to_i
-    @statistics[:total_length] = total_length + content[:length].to_i
+      @statistics[:crawl_counter] = crawl_counter
+      @statistics[:queue_counter] = queue_counter
+      
+      total_length = @statistics[:total_length].to_i
+      @statistics[:total_length] = total_length + content[:length].to_i
 
-    mime_counts = {}
-    if @statistics.has_key? :mime_counts
-      mime_counts = @statistics[:mime_counts]
-      if mime_counts.has_key? content[:mime_type]
-        mime_counts[content[:mime_type]] += 1
+      mime_counts = {}
+      if @statistics.has_key? :mime_counts
+        mime_counts = @statistics[:mime_counts]
+        if mime_counts.has_key? content[:mime_type]
+          mime_counts[content[:mime_type]] += 1
+        else
+          mime_counts[content[:mime_type]] = 1
+        end
       else
-        mime_counts[content[:mime_type]] = 1
+        mime_counts = {content[:mime_type] => 1}
       end
-    else
-      mime_counts = {content[:mime_type] => 1}
-    end
-    @statistics[:mime_counts] = mime_counts.to_json
+      @statistics[:mime_counts] = mime_counts.to_json
 
-    # record mime categories stats
-    if content[:mime_type].cobweb_starts_with? "text"
-      increment_time_stat("mime_text_count")
-    elsif content[:mime_type].cobweb_starts_with? "application"
-      increment_time_stat("mime_application_count")
-    elsif content[:mime_type].cobweb_starts_with? "audio"
-      increment_time_stat("mime_audio_count")
-    elsif content[:mime_type].cobweb_starts_with? "image"
-      increment_time_stat("mime_image_count")
-    elsif content[:mime_type].cobweb_starts_with? "message"
-      increment_time_stat("mime_message_count")
-    elsif content[:mime_type].cobweb_starts_with? "model"
-      increment_time_stat("mime_model_count")
-    elsif content[:mime_type].cobweb_starts_with? "multipart"
-      increment_time_stat("mime_multipart_count")
-    elsif content[:mime_type].cobweb_starts_with? "video"
-      increment_time_stat("mime_video_count")
-    end
-    
-    status_counts = {}
-    if @statistics.has_key? :status_counts
-      status_counts = @statistics[:status_counts]
-      status_code = content[:status_code].to_i.to_s.to_sym
-      if status_counts.has_key? status_code
-        status_counts[status_code] += 1
+      # record mime categories stats
+      if content[:mime_type].cobweb_starts_with? "text"
+        increment_time_stat("mime_text_count")
+      elsif content[:mime_type].cobweb_starts_with? "application"
+        increment_time_stat("mime_application_count")
+      elsif content[:mime_type].cobweb_starts_with? "audio"
+        increment_time_stat("mime_audio_count")
+      elsif content[:mime_type].cobweb_starts_with? "image"
+        increment_time_stat("mime_image_count")
+      elsif content[:mime_type].cobweb_starts_with? "message"
+        increment_time_stat("mime_message_count")
+      elsif content[:mime_type].cobweb_starts_with? "model"
+        increment_time_stat("mime_model_count")
+      elsif content[:mime_type].cobweb_starts_with? "multipart"
+        increment_time_stat("mime_multipart_count")
+      elsif content[:mime_type].cobweb_starts_with? "video"
+        increment_time_stat("mime_video_count")
+      end
+      
+      status_counts = {}
+      if @statistics.has_key? :status_counts
+        status_counts = @statistics[:status_counts]
+        status_code = content[:status_code].to_i.to_s.to_sym
+        if status_counts.has_key? status_code
+          status_counts[status_code] += 1
+        else
+          status_counts[status_code] = 1
+        end      
       else
-        status_counts[status_code] = 1
-      end      
-    else
-      status_counts = {status_code => 1}
-    end
-    
-    # record statistics by status type
-    if content[:status_code] >= 200 && content[:status_code] < 300
-      increment_time_stat("status_200_count")
-    elsif content[:status_code] >= 400 && content[:status_code] < 500
-      increment_time_stat("status|_400_count")
-    elsif content[:status_code] >= 500 && content[:status_code] < 600
-      increment_time_stat("status|_500_count")
-    end
-    
-    @statistics[:status_counts] = status_counts.to_json
-    
-    ## time based statistics
-    increment_time_stat("minute_totals", "minute", 60)
-    
-    redis_command = "@redis.hmset 'statistics', #{@statistics.keys.map{|key| "'#{key}', '#{@statistics[key].to_s.gsub("'","''")}'"}.join(", ")}"
-    instance_eval redis_command
-    
+        status_counts = {status_code => 1}
+      end
+      
+      # record statistics by status type
+      if content[:status_code] >= 200 && content[:status_code] < 300
+        increment_time_stat("status_200_count")
+      elsif content[:status_code] >= 400 && content[:status_code] < 500
+        increment_time_stat("status|_400_count")
+      elsif content[:status_code] >= 500 && content[:status_code] < 600
+        increment_time_stat("status|_500_count")
+      end
+      
+      @statistics[:status_counts] = status_counts.to_json
+      
+      ## time based statistics
+      increment_time_stat("minute_totals", "minute", 60)
+      
+      redis_command = "@redis.hmset 'statistics', #{@statistics.keys.map{|key| "'#{key}', '#{@statistics[key].to_s.gsub("'","''")}'"}.join(", ")}"
+      instance_eval redis_command
+    }
     @statistics
   end
   

data/spec/spec_helper.rb

@@ -12,16 +12,17 @@ RSpec.configure do |config|
   if ENV["TRAVIS_RUBY_VERSION"] || ENV['CI']
     config.filter_run_excluding :local_only => true
   end
+
+  Thread.new do
+    @thin ||= Thin::Server.start("0.0.0.0", 3532, SampleServer.app)
+  end
+
+  # WAIT FOR START TO COMPLETE
+  sleep 1
+
   
   config.before(:all) {
     # START THIN SERVER TO HOST THE SAMPLE SITE FOR CRAWLING
-    @thin = nil
-    Thread.new do
-      @thin = Thin::Server.start("0.0.0.0", 3532, SampleServer.app)
-    end
-  
-    # WAIT FOR START TO COMPLETE
-    sleep 1
   }
   
   config.before(:each) {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-11 00:00:00.000000000 Z
+date: 2013-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70331050084220 !ruby/object:Gem::Requirement
+  requirement: &70128767187740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050084220
+  version_requirements: *70128767187740
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70331050081420 !ruby/object:Gem::Requirement
+  requirement: &70128767183580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050081420
+  version_requirements: *70128767183580
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70331050075780 !ruby/object:Gem::Requirement
+  requirement: &70128767182220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050075780
+  version_requirements: *70128767182220
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70331050066140 !ruby/object:Gem::Requirement
+  requirement: &70128767175380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050066140
+  version_requirements: *70128767175380
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70331050054340 !ruby/object:Gem::Requirement
+  requirement: &70128767172200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050054340
+  version_requirements: *70128767172200
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70331050049640 !ruby/object:Gem::Requirement
+  requirement: &70128767170580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050049640
+  version_requirements: *70128767170580
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70331050048160 !ruby/object:Gem::Requirement
+  requirement: &70128767167800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050048160
+  version_requirements: *70128767167800
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70331050047400 !ruby/object:Gem::Requirement
+  requirement: &70128767161460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050047400
+  version_requirements: *70128767161460
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70331050046440 !ruby/object:Gem::Requirement
+  requirement: &70128767157640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050046440
+  version_requirements: *70128767157640
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70331050045060 !ruby/object:Gem::Requirement
+  requirement: &70128767152300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050045060
+  version_requirements: *70128767152300
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70331050044240 !ruby/object:Gem::Requirement
+  requirement: &70128767148180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,7 +131,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331050044240
+  version_requirements: *70128767148180
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more performant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface