cobweb 0.0.58 → 0.0.59

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.58
+h1. Cobweb v0.0.59
 !https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
 
 h2. Intro 
@@ -38,11 +38,6 @@ h3. Data Returned
     ** :related - url's from link tags
     ** :scripts - url's from script tags
     ** :styles - url's from within link tags with rel of stylesheet and from url() directives with stylesheets
-  * :crawl_id - the id used internally for identifying the crawl.  Can be used by the processing job to seperate crawls
-  * :internal_urls - an array of urls with * wildcards that represent urls internal to the site (ie pages within the same domain)
-  * :external_urls - an array of urls with * wildcards that represent urls external to the site (overrides internal_urls)
-  * :obey_robots - boolean determining if robots.txt should be honoured. (default: false)
-  * :user_agent - user agent string to match in robots.txt (not sent as user_agent of requests yet) (default: cobweb)
   
   The source for the links can be overridden, contact me for the syntax (don't have time to put it into this documentation, will as soon as i have time!)
 
@@ -70,6 +65,13 @@ Creates a new crawler object based on a base_url
     ** :redis_options                 - hash containing the initialization options for redis (e.g. {:host => "redis.mydomain.com"}) (Default: {})
     ** :internal_urls                 - array of strings representing internal url forms for your site (eg: ['http://test.com/*', 'http://blog.test.com/*', 'http://externaltest.com/*']) (Default: [], although your first url's scheme, host and domain are added)
     ** :first_page_redirect_internal  - if true and the first page crawled is a redirect, it will add the final destination of redirects to the internal_urls (e.g. http://www.test.com gets redirected to http://test.com) (Default: true)
+    ** :crawl_id                      - the id used internally for identifying the crawl.  Can be used by the processing job to seperate crawls
+    ** :internal_urls                 - an array of urls with * wildcards that represent urls internal to the site (ie pages within the same domain)
+    ** :external_urls                 - an array of urls with * wildcards that represent urls external to the site (overrides internal_urls)
+    ** :obey_robots                   - boolean determining if robots.txt should be honoured. (default: false)
+    ** :user_agent                    - user agent string to match in robots.txt (not sent as user_agent of requests yet) (default: cobweb)
+    ** :crawl_limit_by_page           - sets the crawl counter to only use html page types when counting objects crawled
+    ** :valid_mime_types              - an array of mime types that takes wildcards (eg 'text/*') defaults to ['*/*']
     
 bq. crawler = CobWeb.new(:follow_redirects => false)
 

data/lib/cobweb.rb

@@ -180,6 +180,7 @@ class Cobweb
             content[:character_set] = charset
           end
           content[:length] = response.content_length
+          content[:text_content] = text_content?(content[:mime_type])
           if text_content?(content[:mime_type])
             if response["Content-Encoding"]=="gzip"
               content[:body] = Zlib::GzipReader.new(StringIO.new(response.body)).read
@@ -389,21 +390,22 @@ class Cobweb
     
   end
   
+  # escapes characters with meaning in regular expressions and adds wildcard expression
+  def self.escape_pattern_for_regex(pattern)
+    pattern = pattern.gsub(".", "\\.")
+    pattern = pattern.gsub("?", "\\?")
+    pattern = pattern.gsub("+", "\\+")
+    pattern = pattern.gsub("*", ".*?")
+    pattern
+  end
+  
   private
   # checks if the mime_type is textual
   def text_content?(content_type)
     @options[:text_mime_types].each do |mime_type|
-      return true if content_type.match(escape_pattern_for_regex(mime_type))
+      return true if content_type.match(Cobweb.escape_pattern_for_regex(mime_type))
     end
     false
   end
-
-  # escapes characters with meaning in regular expressions and adds wildcard expression
-  def escape_pattern_for_regex(pattern)
-    pattern = pattern.gsub(".", "\\.")
-    pattern = pattern.gsub("?", "\\?")
-    pattern = pattern.gsub("*", ".*?")
-    pattern
-  end
   
 end

data/lib/cobweb_crawler.rb

@@ -85,8 +85,6 @@ class CobwebCrawler
               
               @stats.update_statistics(content, crawl_counter, queue_counter)
               @stats.update_status("Completed #{url}.")
-              puts "Crawled: #{crawl_counter.to_i} Limit: #{@options[:crawl_limit].to_i} Queued: #{queue_counter.to_i}" if @debug 
-              
               yield content, @stats.get_statistics if block_given?
 
             rescue => e

data/lib/cobweb_links.rb

@@ -12,8 +12,8 @@ class CobwebLinks
     @options[:external_urls] = [] unless @options.has_key? :external_urls
     @options[:debug] = false unless @options.has_key? :debug
     
-    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{escape_pattern_for_regex(pattern)}")}
-    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{escape_pattern_for_regex(pattern)}")}
+    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern)}")}
+    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern)}")}
     
   end
   
@@ -52,15 +52,6 @@ class CobwebLinks
     @internal_patterns.select{|pattern| link.match(pattern)}.empty? || !@external_patterns.select{|pattern| link.match(pattern)}.empty?
   end
   
-  private
-  # escapes characters with meaning in regular expressions and adds wildcard expression
-  def escape_pattern_for_regex(pattern)
-    pattern = pattern.gsub(".", "\\.")
-    pattern = pattern.gsub("?", "\\?")
-    pattern = pattern.gsub("*", ".*?")
-    ap pattern if @options[:debug]
-    pattern
-  end
 end
 
 # Exception raised for :internal_urls missing from CobwebLinks

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.58"
+    "0.0.59"
   end
 
 end

data/lib/crawl_job.rb

@@ -11,83 +11,105 @@ class CrawlJob
 
   # Resque perform method to maintain the crawl, enqueue found links and detect the end of crawl
   def self.perform(content_request)
-    
     # change all hash keys to symbols
     content_request = HashUtil.deep_symbolize_keys(content_request)
+    @content_request = content_request
     
     content_request[:redis_options] = {} unless content_request.has_key? :redis_options
+    content_request[:crawl_limit_by_page] = false unless content_request.has_key? :crawl_limit_by_page
+    content_request[:valid_mime_types] = ["*/*"] unless content_request.has_key? :valid_mime_types
+    
     @redis = NamespacedRedis.new(content_request[:redis_options], "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
     @stats = Stats.new(content_request)
     
     @debug = content_request[:debug]
     
-    refresh_counters
+    decrement_queue_counter
     
     # check we haven't crawled this url before
     unless @redis.sismember "crawled", content_request[:url]
-      @redis.srem "queued", content_request[:url]
-      decrement_queue_counter
-      @redis.sadd "crawled", content_request[:url]
-      increment_crawl_counter
       
-      # if there is no limit or we're still under it lets get the url
-      if within_crawl_limits?(content_request[:crawl_limit])
+      content = Cobweb.new(content_request).get(content_request[:url], content_request)
 
-        content = Cobweb.new(content_request).get(content_request[:url], content_request)
-        
-        ## update statistics
-        @stats.update_status("Crawling #{content_request[:url]}...")
-        @stats.update_statistics(content)
-        
-        # set the base url if this is the first page
-        set_base_url @redis, content, content_request
+      if is_permitted_type(content)
+        # if there is no limit or we're still under it lets get the url
+        if within_crawl_limits?(content_request[:crawl_limit])
+          #update the queued and crawled lists if we are within the crawl limits.
+          @redis.srem "queued", content_request[:url]
+          @redis.sadd "crawled", content_request[:url]
+          # increment the counter if we are not limiting by page only || we are limiting count by page and it is a page
+          if content_request[:crawl_limit_by_page]
+            if content[:mime_type].match("text/html")
+              increment_crawl_counter
+              increment_crawl_started_counter
+            end
+          else
+            increment_crawl_counter
+            increment_crawl_started_counter 
+          end
         
-        @cobweb_links = CobwebLinks.new(content_request)
-        if within_queue_limits?(content_request[:crawl_limit])
-          internal_links = ContentLinkParser.new(content_request[:url], content[:body]).all_links(:valid_schemes => [:http, :https])
-
-          # select the link if its internal
-          internal_links.select!{|link| @cobweb_links.internal?(link)}
+          ## update statistics
+          @stats.update_status("Crawling #{content_request[:url]}...")
+          @stats.update_statistics(content)
+          
+          # set the base url if this is the first page
+          set_base_url @redis, content, content_request
+          
+          @cobweb_links = CobwebLinks.new(content_request)
+          if within_queue_limits?(content_request[:crawl_limit])
+            internal_links = ContentLinkParser.new(content_request[:url], content[:body]).all_links(:valid_schemes => [:http, :https])
+            
+            # select the link if its internal
+            internal_links.select!{|link| @cobweb_links.internal?(link)}
         
-          # reject the link if we've crawled it or queued it
-          internal_links.reject!{|link| @redis.sismember("crawled", link)}
-          internal_links.reject!{|link| @redis.sismember("queued", link)}
+            # reject the link if we've crawled it or queued it
+            internal_links.reject!{|link| @redis.sismember("crawled", link)}
+            internal_links.reject!{|link| @redis.sismember("queued", link)}
           
-          internal_links.each do |link|
-            enqueue_content(content_request, link) if within_queue_limits?(content_request[:crawl_limit])
+            internal_links.each do |link|
+              enqueue_content(content_request, link) if within_queue_limits?(content_request[:crawl_limit])
+            end
           end
-        end
         
-        # enqueue to processing queue
-        send_to_processing_queue(content, content_request)
+          # enqueue to processing queue
+          send_to_processing_queue(content, content_request)
         
-        #if the enqueue counter has been requested update that
-        if content_request.has_key? :enqueue_counter_key                                                                                  
-          enqueue_redis = NamespacedRedis.new(content_request[:redis_options], content_request[:enqueue_counter_namespace].to_s)
-          current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
-          enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
-        end
-
-        # if there's nothing left queued or the crawled limit has been reached
-        if content_request[:crawl_limit].nil? || content_request[:crawl_limit] == 0
-          if @redis.scard("queued") == 0
-            finished(content_request)
+          #if the enqueue counter has been requested update that
+          if content_request.has_key? :enqueue_counter_key                                                                                  
+            enqueue_redis = NamespacedRedis.new(content_request[:redis_options], content_request[:enqueue_counter_namespace].to_s)
+            current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
+            enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
           end
-        elsif @queue_counter == 0 || @crawl_counter > content_request[:crawl_limit].to_i
-          finished(content_request)
+
+          # update the queue and crawl counts -- doing this very late in the piece so that the following transaction all occurs at once.
+          # really we should do this with a lock https://github.com/PatrickTulskie/redis-lock
+          #increment_crawl_counter
+          puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter} In Progress: #{@crawl_started_counter-@crawl_counter}" if @debug
         end
+      else
+        puts "ignoring #{content_request[:url]} as mime_type is #{content[:mime_type]}" if content_request[:debug]
       end
+      
     else
       @redis.srem "queued", content_request[:url]
-      decrement_queue_counter
       puts "Already crawled #{content_request[:url]}" if content_request[:debug]
     end
-
+    
+    # if there's nothing left queued or the crawled limit has been reached
+    if content_request[:crawl_limit].nil? || content_request[:crawl_limit] == 0
+      if @queue_counter == 0
+        finished(content_request)
+      end
+    elsif @queue_counter == 0 || @crawl_counter >= content_request[:crawl_limit].to_i
+      finished(content_request)
+    end
+    
   end
 
   # Sets the crawl status to 'Crawl Stopped' and enqueues the crawl finished job
   def self.finished(content_request)
     # finished
+    ap "FINISHED"
     @stats.end_crawl(content_request)
     Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))
   end
@@ -95,7 +117,10 @@ class CrawlJob
   # Enqueues the content to the processing queue setup in options
   def self.send_to_processing_queue(content, content_request)
     content_to_send = content.merge({:internal_urls => content_request[:internal_urls], :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]})
-    if content_request[:use_encoding_safe_process_job]
+    if content_request[:direct_call_process_job]
+      clazz = const_get(content_request[:processing_queue])
+      clazz.perform(content_to_send)
+    elsif content_request[:use_encoding_safe_process_job]
       content_to_send[:body] = Base64.encode64(content[:body])
       content_to_send[:processing_queue] = content_request[:processing_queue]
       Resque.enqueue(EncodingSafeProcessJob, content_to_send)
@@ -103,19 +128,28 @@ class CrawlJob
       Resque.enqueue(const_get(content_request[:processing_queue]), content_to_send)
     end
     puts "#{content_request[:url]} has been sent for processing. use_encoding_safe_process_job: #{content_request[:use_encoding_safe_process_job]}" if content_request[:debug]
-    puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter}" if content_request[:debug]
   end
 
   private
   
+  # Helper method to determine if this content is to be processed or not
+  def self.is_permitted_type(content)
+    @content_request[:valid_mime_types].each do |mime_type|
+      return true if content[:mime_type].match(Cobweb.escape_pattern_for_regex(mime_type))
+    end
+    false
+  end
+  
   # Returns true if the crawl count is within limits
   def self.within_crawl_limits?(crawl_limit)
+    refresh_counters
     crawl_limit.nil? or @crawl_counter <= crawl_limit.to_i
+    crawl_limit.nil? or @crawl_started_counter < crawl_limit.to_i
   end
   
   # Returns true if the queue count is calculated to be still within limits when complete
   def self.within_queue_limits?(crawl_limit)
-    within_crawl_limits?(crawl_limit) && (crawl_limit.nil? || (@queue_counter + @crawl_counter) < crawl_limit.to_i)
+    @content_request[:crawl_limit_by_page] || within_crawl_limits?(crawl_limit) && (crawl_limit.nil? || (@queue_counter + @crawl_started_counter) < crawl_limit.to_i)
   end
   
   # Sets the base url in redis.  If the first page is a redirect, it sets the base_url to the destination
@@ -149,6 +183,10 @@ class CrawlJob
     @redis.incr "crawl-counter"
     refresh_counters
   end
+  def self.increment_crawl_started_counter
+    @redis.incr "crawl-started-counter"
+    refresh_counters
+  end
   # Decrements the queue counter and refreshes crawl counters
   def self.decrement_queue_counter
     @redis.decr "queue-counter"
@@ -157,12 +195,20 @@ class CrawlJob
   # Refreshes the crawl counters
   def self.refresh_counters
     @crawl_counter = @redis.get("crawl-counter").to_i
+    @crawl_started_counter = @redis.get("crawl-started-counter").to_i
     @queue_counter = @redis.get("queue-counter").to_i
   end
+  
+  def self.print_counters
+    puts "@crawl_counter: #{@crawl_counter} @crawl_started_counter: #{@crawl_started_counter} @queue_counter: #{@queue_counter}"    
+  end
+  
   # Sets the crawl counters based on the crawled and queued queues
   def self.reset_counters
+    @redis.set("crawl-started-counter", @redis.smembers("crawled").count)
     @redis.set("crawl-counter", @redis.smembers("crawled").count)
     @redis.set("queue-counter", @redis.smembers("queued").count)
+    @crawl_started_counter = @redis.get("crawl-started-counter").to_i
     @crawl_counter = @redis.get("crawl-counter").to_i
     @queue_counter = @redis.get("queue-counter").to_i
   end

data/lib/robots.rb

@@ -28,10 +28,10 @@ class Robots
   def allowed?(url)
     uri = URI.parse(url)
     @params[:allow].each do |pattern|
-      return true if uri.path.match(escape_pattern_for_regex(pattern))
+      return true if uri.path.match(Cobweb.escape_pattern_for_regex(pattern))
     end
     @params[:disallow].each do |pattern|
-      return false if uri.path.match(escape_pattern_for_regex(pattern))
+      return false if uri.path.match(Cobweb.escape_pattern_for_regex(pattern))
     end
     true
   end
@@ -45,13 +45,6 @@ class Robots
   end
   
   private
-  # escapes characters with meaning in regular expressions and adds wildcard expression
-  def escape_pattern_for_regex(pattern)
-    pattern = pattern.gsub(".", "\\.")
-    pattern = pattern.gsub("?", "\\?")
-    pattern = pattern.gsub("*", ".*?")
-    pattern
-  end
   
   def parse_data(data)
     user_agents = {}

data/spec/cobweb/cobweb_job_spec.rb

@@ -46,7 +46,32 @@ describe Cobweb, :local_only => true do
       Resque.size("cobweb_finished_job").should == 1
     end
   end
-  
+  describe "with limited mime_types" do
+    before(:each) do
+      @request = {
+        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+        :quiet => true,
+        :cache => nil,
+        :valid_mime_types => ["text/html"]
+      }
+      @cobweb = Cobweb.new @request
+    end
+      
+    it "should only crawl html pages" do
+      crawl = @cobweb.start(@base_url)
+      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+      wait_for_crawl_finished crawl[:crawl_id]
+      Resque.size("cobweb_process_job").should == 8
+      
+      mime_types = Resque.peek("cobweb_process_job", 0, 100).map{|job| job["args"][0]["mime_type"]}
+      mime_types.count.should == 8
+      mime_types.map{|m| m.should == "text/html"}
+      mime_types.select{|m| m=="text/html"}.count.should == 8
+      
+      
+    end
+    
+  end
   describe "with a crawl limit" do
     before(:each) do
       @request = {
@@ -54,12 +79,12 @@ describe Cobweb, :local_only => true do
         :quiet => true,
         :cache => nil
       }
-      @cobweb = Cobweb.new @request
     end
   
     describe "limit to 1" do
       before(:each) do
         @request[:crawl_limit] = 1
+        @cobweb = Cobweb.new @request
       end
 
       it "should not crawl the entire site" do
@@ -82,11 +107,30 @@ describe Cobweb, :local_only => true do
       end      
     
     end
+
+    describe "for pages only" do
+      before(:each) do
+        @request[:crawl_limit_by_page] = true
+        @request[:crawl_limit] = 5
+        @cobweb = Cobweb.new @request
+      end
+      
+      it "should only use html pages towards the crawl limit" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        mime_types = Resque.peek("cobweb_process_job", 0, 200).map{|job| job["args"][0]["mime_type"]}
+        mime_types.count.should == 70
+        mime_types.select{|m| m=="text/html"}.count.should == 5
+      end
+    end
   
-    describe "limit to 3" do
+    describe "limit to 10" do
       before(:each) do
-        @request[:crawl_limit] = 3
+        @request[:crawl_limit] = 10
+        @cobweb = Cobweb.new @request
       end
+      
       it "should not crawl the entire site" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
@@ -99,21 +143,21 @@ describe Cobweb, :local_only => true do
         wait_for_crawl_finished crawl[:crawl_id]
         Resque.size("cobweb_finished_job").should == 1
       end      
-      it "should only crawl 3 pages" do
+      it "should only crawl 10 objects" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
-        Resque.size("cobweb_process_job").should == 3
-      end      
-    
+        Resque.size("cobweb_process_job").should == 10
+      end
     end
   
     describe "limit to 100" do
       before(:each) do
         @request[:crawl_limit] = 100
+        @cobweb = Cobweb.new @request
       end
-    
-      it "should crawl the entire site" do
+      
+      it "should crawl the entire sample site" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
@@ -138,19 +182,21 @@ describe Cobweb, :local_only => true do
     @all_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
     command = "kill #{(@all_processes - @existing_processes).join(" ")}"
     IO.popen(command)
+    
+    clear_queues
   end
 
 end
 
 def wait_for_crawl_finished(crawl_id, timeout=20)
-  counter = 0 
-  while(running?(crawl_id) && counter < timeout) do
-    sleep 1
-    counter+=1
+  counter = 0
+  start_time = Time.now
+  while(running?(crawl_id) && Time.now < start_time + timeout) do
+    sleep 0.5
   end
-  if counter > timeout
+  if Time.now > start_time + timeout
     raise "End of crawl not detected"
-  end 
+  end
 end
 
 def running?(crawl_id)
@@ -161,9 +207,10 @@ def clear_queues
   Resque.queues.each do |queue|
     Resque.remove_queue(queue)
   end
+  puts "Cleared"
   
   Resque.size("cobweb_process_job").should == 0
-  Resque.size("cobweb_finished_job").should == 0      
+  Resque.size("cobweb_finished_job").should == 0
 end
 
 

data/spec/samples/sample_site/index.html

@@ -71,11 +71,6 @@
 				</ul> 
 			</li> 
 			<li><a href="typography.html">Typography</a></li>
-			<li><a href="boxgrid.html">Boxes Grid</a></li>
-			<li><a href="forms.html">Forms</a></li>
-			<li><a href="gallery.html">Gallery</a></li>
-			<li><a href="tables.html">Tables</a></li>
-			<li><a href="more.html">More</a></li>
 		</ul>
 		<div class="search">
 			<form action="" method="post">

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.58
+  version: 0.0.59
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-30 00:00:00.000000000 Z
+date: 2012-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70328776801460 !ruby/object:Gem::Requirement
+  requirement: &70204470213880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776801460
+  version_requirements: *70204470213880
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70328776799760 !ruby/object:Gem::Requirement
+  requirement: &70204470212220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776799760
+  version_requirements: *70204470212220
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70328776798960 !ruby/object:Gem::Requirement
+  requirement: &70204470211500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776798960
+  version_requirements: *70204470211500
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70328776797840 !ruby/object:Gem::Requirement
+  requirement: &70204470210300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776797840
+  version_requirements: *70204470210300
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70328776796300 !ruby/object:Gem::Requirement
+  requirement: &70204470208860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776796300
+  version_requirements: *70204470208860
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70328776811560 !ruby/object:Gem::Requirement
+  requirement: &70204470223880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776811560
+  version_requirements: *70204470223880
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70328776810940 !ruby/object:Gem::Requirement
+  requirement: &70204470223280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776810940
+  version_requirements: *70204470223280
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70328776810380 !ruby/object:Gem::Requirement
+  requirement: &70204470222720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776810380
+  version_requirements: *70204470222720
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70328776809840 !ruby/object:Gem::Requirement
+  requirement: &70204470222160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328776809840
+  version_requirements: *70204470222160
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70328776809160 !ruby/object:Gem::Requirement
+  requirement: &70204470221480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70328776809160
+  version_requirements: *70204470221480
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface