cobweb 0.0.65 → 0.0.66

data/README.textile

@@ -1,5 +1,6 @@
 
-h1. Cobweb v0.0.65
+h1. Cobweb v0.0.66
+
 !https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
 
 h2. Intro 

data/lib/cobweb.rb

@@ -45,7 +45,7 @@ class Cobweb
     default_first_page_redirect_internal_to   true
     default_text_mime_types_to                ["text/*", "application/xhtml+xml"]
     default_obey_robots_to                    false
-    default_user_agent_to                     "cobweb"
+    default_user_agent_to                     "cobweb/#{Cobweb.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"
     
   end
   

data/lib/cobweb_crawler.rb

@@ -27,7 +27,7 @@ class CobwebCrawler
     
     @stats = Stats.new(@options.merge(:crawl_id => @crawl_id))
     if @options[:web_statistics]
-      Server.start
+      Server.start(@options)
     end
     
     @cobweb = Cobweb.new(@options)

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.65"
+    "0.0.66"
   end
 
 end

data/lib/crawl_job.rb

@@ -24,12 +24,11 @@ class CrawlJob
     
     @debug = content_request[:debug]
     
-    decrement_queue_counter
-
     # check we haven't crawled this url before
     unless @redis.sismember "crawled", content_request[:url]
       # if there is no limit or we're still under it lets get the url
       if within_crawl_limits?(content_request[:crawl_limit])
+        puts "cbpl: #{content_request[:url]}" if content_request[:crawl_limit_by_page]
         content = Cobweb.new(content_request).get(content_request[:url], content_request)
         if content_request[:url] == @redis.get("original_base_url")
            @redis.set("crawled_base_url", content[:base_url])
@@ -78,7 +77,7 @@ class CrawlJob
             send_to_processing_queue(content, content_request)
 
             #if the enqueue counter has been requested update that
-            if content_request.has_key? :enqueue_counter_key
+            if content_request.has_key?(:enqueue_counter_key)
               enqueue_redis = NamespacedRedis.new(content_request[:redis_options], content_request[:enqueue_counter_namespace].to_s)
               current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
               enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
@@ -89,14 +88,16 @@ class CrawlJob
 
             # update the queue and crawl counts -- doing this very late in the piece so that the following transaction all occurs at once.
             # really we should do this with a lock https://github.com/PatrickTulskie/redis-lock
+            # stewart: i'm looking at the layout of this, think that there is scope for cleaning up the perform method to be more DRY.
             if content_request[:crawl_limit_by_page]
               if content[:mime_type].match("text/html")
                 increment_crawl_counter
+                ap "clbp: #{crawl_counter}"
               end
             else
               increment_crawl_counter
             end
-            puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter} In Progress: #{@crawl_started_counter-@crawl_counter}" if @debug
+            puts "Crawled: #{crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{queue_counter} In Progress: #{crawl_started_counter-crawl_counter}" if @debug
           end
         else
           puts "ignoring #{content_request[:url]} as mime_type is #{content[:mime_type]}" if content_request[:debug]
@@ -110,13 +111,15 @@ class CrawlJob
       puts "Already crawled #{content_request[:url]}" if content_request[:debug]
     end
     
+    decrement_queue_counter
+    puts content_request[:crawl_limit]
+    print_counters
     # if there's nothing left queued or the crawled limit has been reached
-    refresh_counters
     if content_request[:crawl_limit].nil? || content_request[:crawl_limit] == 0
-      if @queue_counter+@crawl_started_counter-@crawl_counter == 0
+      if queue_counter + crawl_started_counter - crawl_counter == 0
         finished(content_request)
       end
-    elsif (@queue_counter +@crawl_started_counter-@crawl_counter)== 0 || @crawl_counter >= content_request[:crawl_limit].to_i
+    elsif (queue_counter+crawl_started_counter-crawl_counter)== 0 || crawl_counter >= content_request[:crawl_limit].to_i
       finished(content_request)
     end
     
@@ -167,14 +170,12 @@ class CrawlJob
   
   # Returns true if the crawl count is within limits
   def self.within_crawl_limits?(crawl_limit)
-    refresh_counters
-    crawl_limit.nil? or @crawl_started_counter < crawl_limit.to_i
+    crawl_limit.nil? or crawl_counter < crawl_limit.to_i
   end
   
   # Returns true if the queue count is calculated to be still within limits when complete
   def self.within_queue_limits?(crawl_limit)
-    refresh_counters
-    (@content_request[:crawl_limit_by_page]&& (crawl_limit.nil? or @crawl_counter < crawl_limit.to_i)) || within_crawl_limits?(crawl_limit) && (crawl_limit.nil? || (@queue_counter + @crawl_counter) < crawl_limit.to_i)
+    (@content_request[:crawl_limit_by_page]&& (crawl_limit.nil? or crawl_counter < crawl_limit.to_i)) || within_crawl_limits?(crawl_limit) && (crawl_limit.nil? || (queue_counter + crawl_counter) < crawl_limit.to_i)
   end
   
   # Sets the base url in redis.  If the first page is a redirect, it sets the base_url to the destination
@@ -201,27 +202,27 @@ class CrawlJob
   # Increments the queue counter and refreshes crawl counters
   def self.increment_queue_counter
     @redis.incr "queue-counter"
-    refresh_counters
   end
   # Increments the crawl counter and refreshes crawl counters
   def self.increment_crawl_counter
     @redis.incr "crawl-counter"
-    refresh_counters
   end
   def self.increment_crawl_started_counter
     @redis.incr "crawl-started-counter"
-    refresh_counters
   end
   # Decrements the queue counter and refreshes crawl counters
   def self.decrement_queue_counter
     @redis.decr "queue-counter"
-    refresh_counters
   end
-  # Refreshes the crawl counters
-  def self.refresh_counters
-    @crawl_counter = @redis.get("crawl-counter").to_i
-    @crawl_started_counter = @redis.get("crawl-started-counter").to_i
-    @queue_counter = @redis.get("queue-counter").to_i
+  
+  def self.crawl_counter
+    @redis.get("crawl-counter").to_i
+  end
+  def self.crawl_started_counter
+    @redis.get("crawl-started-counter").to_i
+  end
+  def self.queue_counter
+    @redis.get("queue-counter").to_i
   end
   
   def self.print_counters
@@ -229,7 +230,7 @@ class CrawlJob
   end
 
   def self.counters
-    "@crawl_counter: #{@crawl_counter} @crawl_started_counter: #{@crawl_started_counter} @queue_counter: #{@queue_counter}"
+    "crawl_counter: #{crawl_counter} crawl_started_counter: #{crawl_started_counter} queue_counter: #{queue_counter}"
   end
 
 end

data/lib/server.rb

@@ -10,17 +10,18 @@ class Server < Sinatra::Base
   
   # Sinatra Dashboard
   get '/' do
-    @full_redis = Redis.new
-    
+    @full_redis = Redis.new(redis_options)
     @colors = ["#00366f", "#006ba0", "#3F0BDB", "#396CB3"]
     
     @crawls = []
-    @full_redis.smembers("cobweb_crawls").each do |crawl_id|
-      redis = NamespacedRedis.new({}, "cobweb-#{Cobweb.version}-#{crawl_id}")
+    @full_redis.smembers("cobweb_crawls").each do |crawl_id|      
+      version = cobweb_version(crawl_id)
+      redis = NamespacedRedis.new(redis_options, "cobweb-#{version}-#{crawl_id}")
       stats = HashUtil.deep_symbolize_keys({
-        :crawl_details => redis.hgetall("crawl_details"), 
+        :cobweb_version => version,
+        :crawl_details => redis.hgetall("crawl_details"),
         :statistics => redis.hgetall("statistics"),
-        :minute_totals => redis.hgetall("minute_totals")
+        :minute_totals => redis.hgetall("minute_totals"),
         })
       @crawls << stats
     end
@@ -30,7 +31,9 @@ class Server < Sinatra::Base
   
   # Sinatra Crawl Detail
   get '/statistics/:crawl_id' do
-    redis = NamespacedRedis.new({}, "cobweb-#{Cobweb.version}-#{params[:crawl_id]}")
+    
+    version = cobweb_version(params[:crawl_id])
+    redis = NamespacedRedis.new(redis_options, "cobweb-#{version}-#{params[:crawl_id]}")
     
     @statistics = HashUtil.deep_symbolize_keys(redis.hgetall("statistics"))
     if @statistics[:status_counts].nil?
@@ -44,6 +47,7 @@ class Server < Sinatra::Base
       @statistics[:mime_counts] = JSON.parse(@statistics[:mime_counts])
     end
     @crawl = {
+        :cobweb_version => version,
         :statistics => @statistics,
         :crawl_details => HashUtil.deep_symbolize_keys(redis.hgetall("crawl_details")), 
         :minute_totals => HashUtil.deep_symbolize_keys(redis.hgetall("minute_totals")),
@@ -56,22 +60,49 @@ class Server < Sinatra::Base
         :pages_count => HashUtil.deep_symbolize_keys(redis.hgetall("pages_count")),
         :assets_count => HashUtil.deep_symbolize_keys(redis.hgetall("assets_count"))
     }
-    ap @crawl
+    (1..30).each do |minutes|
+      date = (DateTime.now.new_offset(0) - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
+    end
+    
     haml :statistics
   end
   
+  def cobweb_version(crawl_id)
+    redis = Redis.new(redis_options)
+    key = redis.keys("cobweb-*-#{crawl_id}-crawl_details").first
+    key =~ /cobweb-(.*?)-(.*?)-crawl_details/
+    cobweb_version = $1
+  end
+  
+  def redis_options
+    Server.cobweb_options[:redis_options]
+  end
+  
   # Starts the Sinatra server, and kills the processes when shutdown
-  def self.start
+  def self.start(options={})
+    @options = options
+    @options[:redis_options] = {} unless @options.has_key? :redis_options
+    ap @options
     unless Server.running?
-      thread = Thread.new do
-        puts "Starting Sinatra"
+      if @options[:run_as_server]
+        puts "Starting Sinatra for cobweb v#{Cobweb.version}"
         Server.run!
         puts "Stopping crawl..."
-        ## we need to manually kill the main thread as sinatra traps the interrupts
-        Thread.main.kill
+      else
+        thread = Thread.new do
+          puts "Starting Sinatra"
+          Server.run!
+          puts "Stopping crawl..."
+          ## we need to manually kill the main thread as sinatra traps the interrupts
+          Thread.main.kill
+        end
       end
     end    
-  end  
+  end
+  
+  def self.cobweb_options
+    @options
+  end
   
 end
 
@@ -81,7 +112,6 @@ class Numeric
   #Returns a human readable format for a number representing a data size
   def to_human
     units = %w{B KB MB GB TB}
-    ap self
     e = 0
     e = (Math.log(self)/Math.log(1024)).floor unless self==0
     s = "%.3f" % (to_f / 1024**e)

data/lib/stats.rb

@@ -1,4 +1,3 @@
-
 # Stats class is the main statisitics hub for monitoring crawls.  Either can be viewed through the Sinatra interface, or returned from the CobwebCrawler.crawl method or block
 class Stats
   

data/spec/cobweb/cobweb_job_spec.rb

@@ -9,7 +9,7 @@ describe Cobweb, :local_only => true do
     # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES
     puts "Starting Workers... Please Wait..."
     `mkdir log`
-    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=5 QUEUE=cobweb_crawl_job > log/output.log &")
+    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=1 QUEUE=cobweb_crawl_job > log/output.log &")
     puts "Workers Started."
   
   end
@@ -17,61 +17,58 @@ describe Cobweb, :local_only => true do
   before(:each) do
     @base_url = "http://localhost:3532/"
     @base_page_count = 77
-  
     clear_queues
   end
-
+  
   describe "with no crawl limit" do
     before(:each) do
-      @request = {
-        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
-        :crawl_limit => nil,
-        :quiet => false,
-        :debug => false,
-        :cache => nil
-      }
-      @cobweb = Cobweb.new @request
+       @request = {
+         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+         :crawl_limit => nil,
+         :quiet => false,
+         :debug => false,
+         :cache => nil
+       }
+       @cobweb = Cobweb.new @request
     end
-  
+   
     it "should crawl entire site" do
-      crawl = @cobweb.start(@base_url)
-      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
-      wait_for_crawl_finished crawl[:crawl_id]
-      Resque.size("cobweb_process_job").should == @base_page_count
+       crawl = @cobweb.start(@base_url)
+       @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+       wait_for_crawl_finished crawl[:crawl_id]
+       Resque.size("cobweb_process_job").should == @base_page_count
     end
-    it "detect crawl finished" do
-      crawl = @cobweb.start(@base_url)
-      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
-      wait_for_crawl_finished crawl[:crawl_id]
-      Resque.size("cobweb_finished_job").should == 1
+    it "detect crawl finished once" do
+       crawl = @cobweb.start(@base_url)
+       @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+       wait_for_crawl_finished crawl[:crawl_id]
+       Resque.size("cobweb_finished_job").should == 1
     end
   end
-  describe "with limited mime_types" do
-    before(:each) do
-      @request = {
-        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
-        :quiet => true,
-        :cache => nil,
-        :valid_mime_types => ["text/html"]
-      }
-      @cobweb = Cobweb.new @request
-    end
-      
-    it "should only crawl html pages" do
-      crawl = @cobweb.start(@base_url)
-      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
-      wait_for_crawl_finished crawl[:crawl_id]
-      Resque.size("cobweb_process_job").should == 8
-      
-      mime_types = Resque.peek("cobweb_process_job", 0, 100).map{|job| job["args"][0]["mime_type"]}
-      mime_types.count.should == 8
-      mime_types.map{|m| m.should == "text/html"}
-      mime_types.select{|m| m=="text/html"}.count.should == 8
-      
-      
-    end
-    
-  end
+   describe "with limited mime_types" do
+     before(:each) do
+       @request = {
+         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+         :quiet => true,
+         :cache => nil,
+         :valid_mime_types => ["text/html"]
+       }
+       @cobweb = Cobweb.new @request
+     end
+     
+     it "should only crawl html pages" do
+       crawl = @cobweb.start(@base_url)
+       @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+       wait_for_crawl_finished crawl[:crawl_id]
+       Resque.size("cobweb_process_job").should == 8
+     
+       mime_types = Resque.peek("cobweb_process_job", 0, 100).map{|job| job["args"][0]["mime_type"]}
+       mime_types.count.should == 8
+       mime_types.map{|m| m.should == "text/html"}
+       mime_types.select{|m| m=="text/html"}.count.should == 8
+     end
+   
+   end
   describe "with a crawl limit" do
     before(:each) do
       @request = {
@@ -86,7 +83,7 @@ describe Cobweb, :local_only => true do
         @request[:crawl_limit] = 1
         @cobweb = Cobweb.new @request
       end
-
+    
       it "should not crawl the entire site" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
@@ -99,13 +96,12 @@ describe Cobweb, :local_only => true do
         wait_for_crawl_finished crawl[:crawl_id]
         Resque.size("cobweb_process_job").should == 1
       end      
-      it "should notify of crawl finished" do
+      it "should notify of crawl finished once" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
         Resque.size("cobweb_finished_job").should == 1
-      end      
-    
+      end    
     end
 
     describe "for pages only" do
@@ -114,17 +110,17 @@ describe Cobweb, :local_only => true do
         @request[:crawl_limit] = 5
         @cobweb = Cobweb.new @request
       end
-      
+  
       it "should only use html pages towards the crawl limit" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
         mime_types = Resque.peek("cobweb_process_job", 0, 200).map{|job| job["args"][0]["mime_type"]}
-        mime_types.count.should == 70
+        Resque.peek("cobweb_process_job", 0, 200).count.should > 5
         mime_types.select{|m| m=="text/html"}.count.should == 5
       end
     end
-  
+
     describe "limit to 10" do
       before(:each) do
         @request[:crawl_limit] = 10
@@ -137,7 +133,7 @@ describe Cobweb, :local_only => true do
         wait_for_crawl_finished crawl[:crawl_id]
         Resque.size("cobweb_process_job").should_not == @base_page_count
       end      
-      it "should notify of crawl finished" do
+      it "should notify of crawl finished once" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
@@ -150,7 +146,7 @@ describe Cobweb, :local_only => true do
         Resque.size("cobweb_process_job").should == 10
       end
     end
-  
+    
     describe "limit to 100" do
       before(:each) do
         @request[:crawl_limit] = 100
@@ -163,7 +159,7 @@ describe Cobweb, :local_only => true do
         wait_for_crawl_finished crawl[:crawl_id]
         Resque.size("cobweb_process_job").should == @base_page_count
       end      
-      it "should notify of crawl finished" do
+      it "should notify of crawl finished once" do
         crawl = @cobweb.start(@base_url)
         @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
         wait_for_crawl_finished crawl[:crawl_id]
@@ -175,12 +171,13 @@ describe Cobweb, :local_only => true do
         wait_for_crawl_finished crawl[:crawl_id]
         Resque.size("cobweb_process_job").should_not == 100
       end      
-    end    
+    end
   end
 
   after(:all) do
+    
     @all_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
-    command = "kill #{(@all_processes - @existing_processes).join(" ")}"
+    command = "kill -9 #{(@all_processes - @existing_processes).join(" ")}"
     IO.popen(command)
     
     clear_queues
@@ -210,6 +207,7 @@ def clear_queues
   
   Resque.size("cobweb_process_job").should == 0
   Resque.size("cobweb_finished_job").should == 0
+  Resque.peek("cobweb_process_job", 0, 200).should be_empty
 end
 
 

data/views/home.haml

@@ -40,6 +40,7 @@
             %thead
               %tr
                 %th Base URL
+                %th Cobweb Version
                 %th Total Pages
                 %th Total Assets
                 %th Queued Objects
@@ -48,6 +49,7 @@
                 %tr
                   %td
                     %a{:href => "/statistics/#{crawl[:crawl_details][:crawl_id]}"}= crawl[:crawl_details][:base_url]
+                  %td= crawl[:cobweb_version]
                   %td= crawl[:statistics][:page_count]
                   %td= crawl[:statistics][:asset_count]
                   %td= crawl[:statistics][:queue_counter]

data/views/statistics.haml

@@ -40,7 +40,7 @@
 
           %tbody
             - (1..30).each do |minutes|
-              - date = (DateTime.now - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
+              - date = (DateTime.now.new_offset(0) - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
               %tr
                 %th= minutes
                 %td= @crawl[:pages_count][date]
@@ -88,7 +88,7 @@
 
           %tbody
             - (1..30).each do |minutes|
-              - date = (DateTime.now - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
+              - date = (DateTime.now.new_offset(0) - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
               %tr
                 %th= minutes
                 %td= @crawl[:status_200_count][date]
@@ -113,7 +113,7 @@
 
           %tbody
             - (1..30).each do |minutes|
-              - date = (DateTime.now - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
+              - date = (DateTime.now.new_offset(0) - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
               %tr
                 %th= minutes
                 %td= @crawl[:mime_text_count][date]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.65
+  version: 0.0.66
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-26 00:00:00.000000000 Z
+date: 2012-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70202662972080 !ruby/object:Gem::Requirement
+  requirement: &70145280967560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662972080
+  version_requirements: *70145280967560
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70202662971540 !ruby/object:Gem::Requirement
+  requirement: &70145280966480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662971540
+  version_requirements: *70145280966480
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70202662970680 !ruby/object:Gem::Requirement
+  requirement: &70145280965880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662970680
+  version_requirements: *70145280965880
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70202662969640 !ruby/object:Gem::Requirement
+  requirement: &70145280964660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662969640
+  version_requirements: *70145280964660
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70202662968800 !ruby/object:Gem::Requirement
+  requirement: &70145280964040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662968800
+  version_requirements: *70145280964040
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70202662967880 !ruby/object:Gem::Requirement
+  requirement: &70145280963260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662967880
+  version_requirements: *70145280963260
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70202662966760 !ruby/object:Gem::Requirement
+  requirement: &70145280962560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662966760
+  version_requirements: *70145280962560
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70202662961760 !ruby/object:Gem::Requirement
+  requirement: &70145280961780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662961760
+  version_requirements: *70145280961780
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70202662960800 !ruby/object:Gem::Requirement
+  requirement: &70145280960840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70202662960800
+  version_requirements: *70145280960840
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70202662958560 !ruby/object:Gem::Requirement
+  requirement: &70145280960100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70202662958560
+  version_requirements: *70145280960100
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface