cobweb 0.0.13 → 0.0.17

data/lib/cobweb.rb

@@ -1,4 +1,5 @@
 require 'rubygems'
+require 'bundler/setup'
 require 'uri'
 require 'resque'
 require "addressable/uri"
@@ -9,7 +10,7 @@ Dir[File.dirname(__FILE__) + '/*.rb'].each do |file|
   require [File.dirname(__FILE__), File.basename(file, File.extname(file))].join("/")
 end
 
-class CobWeb
+class Cobweb
 
   ## TASKS
   
@@ -65,7 +66,7 @@ class CobWeb
     if redis.get(unique_id) and @options[:cache]
       puts "Cache hit for #{url}" unless @options[:quiet]
       content = JSON.parse(redis.get(unique_id)).deep_symbolize_keys
-      content[:body] = Base64.decode64(content[:body]) unless content[:body].nil? or content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
+      content[:body] = Base64.decode64(content[:body])
       
       content
     else
@@ -93,13 +94,25 @@ class CobWeb
         
         if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
           puts "redirected... " unless @options[:quiet]
+          
+          # get location to redirect to
           url = absolutize.url(response['location']).to_s
+          
+          # decrement redirect limit
           redirect_limit = redirect_limit - 1
+
+          # raise exception if we're being redirected to somewhere we've been redirected to in this content request          
+          #raise RedirectError("Loop detected in redirect for - #{url}") if content[:redirect_through].include? url
+          
+          # raise exception if redirect limit has reached 0
+          raise RedirectError, "Redirect Limit reached" if redirect_limit == 0
+
+          # get the content from redirect location
           content = get(url, redirect_limit)
           content[:url] = uri.to_s
           content[:redirect_through] = [] if content[:redirect_through].nil?
           content[:redirect_through].insert(0, url)
-          
+        
           content[:response_time] = Time.now.to_f - request_time
         else
           content[:response_time] = Time.now.to_f - request_time
@@ -109,8 +122,9 @@ class CobWeb
           # create the content container
           content[:url] = uri.to_s
           content[:status_code] = response.code.to_i
-          content[:mime_type] = response.content_type.split(";")[0].strip
-          if response["Content-Type"].include? ";"
+          content[:mime_type] = ""
+          content[:mime_type] = response.content_type.split(";")[0].strip unless response.content_type.nil?
+          if !response["Content-Type"].nil? && response["Content-Type"].include?(";")
             charset = response["Content-Type"][response["Content-Type"].index(";")+2..-1] if !response["Content-Type"].nil? and response["Content-Type"].include?(";")
             charset = charset[charset.index("=")+1..-1] if charset and charset.include?("=")
             content[:character_set] = charset
@@ -130,10 +144,25 @@ class CobWeb
         end
         # add content to cache if required
         if @options[:cache]
-          content[:body] = Base64.encode64(content[:body]) unless content[:body].nil? or content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
+          content[:body] = Base64.encode64(content[:body])
           redis.set(unique_id, content.to_json)
           redis.expire unique_id, @options[:cache].to_i
         end
+      rescue RedirectError => e
+        puts "ERROR: #{e.message}"
+        
+        ## generate a blank content
+        content = {}
+        content[:url] = uri.to_s
+        content[:response_time] = Time.now.to_f - request_time
+        content[:status_code] = 0
+        content[:length] = 0
+        content[:body] = ""
+        content[:error] = e.message
+        content[:mime_type] = "error/dnslookup"
+        content[:headers] = {}
+        content[:links] = {}
+        
       rescue SocketError => e
         puts "ERROR: #{e.message}"
         
@@ -273,25 +302,7 @@ class CobWeb
   end
 end
 
-## add symbolize methods to hash
-class Hash
-  def symbolize_keys
-    keys.each do |key|
-      if key.instance_of? String
-        value = self[key]
-        self.delete(key)
-        self[key.to_sym] = value        
-      end
-    end
-    self
-  end  
-  def deep_symbolize_keys
-    symbolize_keys
-    keys.each do |key|
-      if self[key].instance_of? Hash
-        self[key].deep_symbolize_keys
-      end
-    end
-    self
-  end
-end
+
+
+
+

data/lib/cobweb_crawler.rb

@@ -1,13 +1,20 @@
+require 'digest/md5'
+require 'date'
+require 'ap'
+
 class CobwebCrawler
   
   def initialize(options={})
     @options = options
     
     @statistic = {}
-    @queue = []
-    @crawled = []
     
-    @cobweb = CobWeb.new(@options)
+    @options[:redis_options] = {:host => "127.0.0.1"} unless @options.has_key? :redis_options
+    crawl_id = Digest::MD5.hexdigest(DateTime.now.inspect.to_s)
+  
+    @redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{crawl_id}")
+    
+    @cobweb = Cobweb.new(@options)
   end
   
   def crawl(base_url, crawl_options = {}, &block)
@@ -17,104 +24,122 @@ class CobwebCrawler
     
     @absolutize = Absolutize.new(@options[:base_url], :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
     
-    crawl_counter = @crawled.count
-    
-    @queue << base_url
-    
-    while !@queue.empty? && (@options[:crawl_limit].to_i == 0 || @options[:crawl_limit].to_i > crawl_counter)
-      
-      url = @queue.first
-      @options[:url] = url
-      unless @crawled.include?(url) || url =~ /\/(.+?)\/\1\/\1/      
-        begin
-          content = @cobweb.get(@options[:url])
+    @redis.sadd "queued", base_url
+    crawl_counter = @redis.scard("crawled").to_i
+    queue_counter = @redis.scard("queued").to_i
 
-          if @statistic[:average_response_time].nil?
-            @statistic[:average_response_time] = content[:response_time].to_f
-          else
-            @statistic[:average_response_time] = (((@statistic[:average_response_time] * crawl_counter) + content[:response_time].to_f) / (crawl_counter + 1))
-          end
+    while queue_counter>0 && (@options[:crawl_limit].to_i == 0 || @options[:crawl_limit].to_i > crawl_counter)      
+      thread = Thread.new do
+
+        url = @redis.spop "queued"
+        crawl_counter = @redis.scard("crawled").to_i
+        queue_counter = @redis.scard("queued").to_i
+        
+        @options[:url] = url
+        unless @redis.sismember("crawled", url.to_s)
+          begin
+            Stats.update_status("Requesting #{url}...")
+            content = @cobweb.get(url)
+            Stats.update_status("Processing #{url}...")
+
+            if @statistic[:average_response_time].nil?
+              @statistic[:average_response_time] = content[:response_time].to_f
+            else
+              @statistic[:average_response_time] = (((@statistic[:average_response_time] * crawl_counter) + content[:response_time].to_f) / (crawl_counter + 1))
+            end
       
-          @statistic[:maximum_response_time] = content[:response_time] if @statistic[:maximum_response_time].nil? || @statistic[:maximum_response_time] < content[:response_time]
-          @statistic[:minimum_response_time] = content[:response_time] if @statistic[:minimum_response_time].nil? || @statistic[:minimum_response_time] > content[:response_time]
+            @statistic[:maximum_response_time] = content[:response_time] if @statistic[:maximum_response_time].nil? || @statistic[:maximum_response_time] < content[:response_time]
+            @statistic[:minimum_response_time] = content[:response_time] if @statistic[:minimum_response_time].nil? || @statistic[:minimum_response_time] > content[:response_time]
       
-          if @statistic[:average_length]
-            @statistic[:average_length] = (((@statistic[:average_length].to_i*crawl_counter) + content[:length].to_i) / (crawl_counter + 1))
-          else
-            @statistic[:average_length] = content[:length].to_i
-          end
+            if @statistic[:average_length]
+              @statistic[:average_length] = (((@statistic[:average_length].to_i*crawl_counter) + content[:length].to_i) / (crawl_counter + 1))
+            else
+              @statistic[:average_length] = content[:length].to_i
+            end
       
-          @statistic[:maximum_length] = content[:length].to_i if @statistic[:maximum_length].nil? || content[:length].to_i > @statistic[:maximum_length].to_i
-          @statistic[:minimum_length] = content[:length].to_i if @statistic[:minimum_length].nil? || content[:length].to_i < @statistic[:minimum_length].to_i
-          @statistic[:total_length] = @statistic[:total_length].to_i + content[:length].to_i
-
-          if content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
-            @statistic[:page_count] = @statistic[:page_count].to_i + 1
-            @statistic[:page_size] = @statistic[:page_count].to_i + content[:length].to_i
-          else
-            @statistic[:asset_count] = @statistic[:asset_count].to_i + 1
-            @statistic[:asset_size] = @statistic[:asset_count].to_i + content[:length].to_i
-          end
+            @statistic[:maximum_length] = content[:length].to_i if @statistic[:maximum_length].nil? || content[:length].to_i > @statistic[:maximum_length].to_i
+            @statistic[:minimum_length] = content[:length].to_i if @statistic[:minimum_length].nil? || content[:length].to_i < @statistic[:minimum_length].to_i
+            @statistic[:total_length] = @statistic[:total_length].to_i + content[:length].to_i
 
-          mime_counts = {}
-          if @statistic.has_key? :mime_counts
-            mime_counts = @statistic[:mime_counts]
-            if mime_counts.has_key? content[:mime_type]
-              mime_counts[content[:mime_type]] += 1
+            if content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
+              @statistic[:page_count] = @statistic[:page_count].to_i + 1
+              @statistic[:page_size] = @statistic[:page_size].to_i + content[:length].to_i
             else
-              mime_counts[content[:mime_type]] = 1
+              @statistic[:asset_count] = @statistic[:asset_count].to_i + 1
+              @statistic[:asset_size] = @statistic[:asset_size].to_i + content[:length].to_i
             end
-          else
-            mime_counts = {content[:mime_type] => 1}
-          end
-          @statistic[:mime_counts] = mime_counts
+            
+            @statistic[:total_redirects] = 0 if @statistic[:total_redirects].nil?
+            @statistic[:total_redirects] += content[:redirect_through].count unless content[:redirect_through].nil?
+            
+            @statistic[:crawl_counter] = crawl_counter
+            @statistic[:queue_counter] = queue_counter
+            
+            mime_counts = {}
+            if @statistic.has_key? :mime_counts
+              mime_counts = @statistic[:mime_counts]
+              if mime_counts.has_key? content[:mime_type]
+                mime_counts[content[:mime_type]] += 1
+              else
+                mime_counts[content[:mime_type]] = 1
+              end
+            else
+              mime_counts = {content[:mime_type] => 1}
+            end
+            @statistic[:mime_counts] = mime_counts
 
-          status_counts = {}
+            status_counts = {}
           
-          if @statistic.has_key? :status_counts
-            status_counts = @statistic[:status_counts]
-            if status_counts.has_key? content[:status_code].to_i
-              status_counts[content[:status_code].to_i] += 1
+            if @statistic.has_key? :status_counts
+              status_counts = @statistic[:status_counts]
+              if status_counts.has_key? content[:status_code].to_i
+                status_counts[content[:status_code].to_i] += 1
+              else
+                status_counts[content[:status_code].to_i] = 1
+              end
             else
-              status_counts[content[:status_code].to_i] = 1
+              status_counts = {content[:status_code].to_i => 1}
             end
-          else
-            status_counts = {content[:status_code].to_i => 1}
-          end
-          @statistic[:status_counts] = status_counts
+            @statistic[:status_counts] = status_counts
 
-          @crawled << url
-          crawl_counter += 1
-          @queue.delete(url)
-          content[:links].keys.map{|key| content[:links][key]}.flatten.each do |link|
-            unless @crawled.include? link
-              puts "Checking if #{link} matches #{@options[:base_url]} as internal?" if @options[:debug]
-              if link.to_s.match(Regexp.new("^#{@options[:base_url]}"))
-                puts "Matched as #{link} as internal" if @options[:debug]
-                unless @crawled.include? link.to_s or @queue.include? link.to_s
-                  puts "Added #{link.to_s} to queue" if @options[:debug] 
-                  @queue << link.to_s
+            @redis.sadd "crawled", url.to_s
+            @redis.incr "crawl-counter" 
+            
+            content[:links].keys.map{|key| content[:links][key]}.flatten.each do |content_link|
+              link = content_link.to_s
+              unless @redis.sismember("crawled", link)
+                puts "Checking if #{link} matches #{@options[:base_url]} as internal?" if @options[:debug]
+                if link.to_s.match(Regexp.new("^#{@options[:base_url]}"))
+                  puts "Matched as #{link} as internal" if @options[:debug]
+                  unless @redis.sismember("crawled", link) || @redis.sismember("queued", link)
+                    puts "Added #{link.to_s} to queue" if @options[:debug]
+                    @redis.sadd "queued", link
+                    crawl_counter = @redis.scard("crawled").to_i
+                    queue_counter = @redis.scard("queued").to_i
+                  end
                 end
               end
             end
-          end
-          @queue.uniq!
             
-          puts "Crawled: #{crawl_counter} Limit: #{@options[:crawl_limit]} Queued: #{@queue.count}" if @options[:debug]
-          
-          yield content if block_given?
+            crawl_counter = @redis.scard("crawled").to_i
+            queue_counter = @redis.scard("queued").to_i
+            Stats.update_statistics(@statistic)
+            Stats.update_status("Completed #{url}.")
+            puts "Crawled: #{crawl_counter.to_i} Limit: #{@options[:crawl_limit].to_i} Queued: #{queue_counter.to_i}" if @options[:debug] 
+       
+            yield content, @statistic if block_given?
 
-        rescue => e
-          puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
-          ap e
-          @queue.delete(url)
-          
+          rescue => e
+            puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
+            ap e
+            ap e.backtrace
+          end
+        else
+          puts "Already crawled #{@options[:url]}" if @options[:debug]
         end
-      else
-        puts "Already crawled #{@options[:url]}" if @options[:debug]
       end
+      thread.join
     end
     @statistic
   end
-  
-end
+end

data/lib/crawl_job.rb

@@ -40,7 +40,7 @@ class CrawlJob
       redis.incr "crawl-counter"
       crawl_counter += 1
       if crawl_counter <= content_request[:crawl_limit].to_i
-        content = CobWeb.new(content_request).get(content_request[:url])
+        content = Cobweb.new(content_request).get(content_request[:url])
 
         ## update statistics
         if redis.hexists "statistics", "average_response_time"

data/lib/hash.rb

@@ -0,0 +1,22 @@
+## add symbolize methods to hash
+class Hash
+  def symbolize_keys
+    keys.each do |key|
+      if key.instance_of? String
+        value = self[key]
+        self.delete(key)
+        self[key.to_sym] = value        
+      end
+    end
+    self
+  end  
+  def deep_symbolize_keys
+    symbolize_keys
+    keys.each do |key|
+      if self[key].instance_of? Hash
+        self[key].deep_symbolize_keys
+      end
+    end
+    self
+  end
+end

data/lib/namespaced_redis.rb

@@ -17,8 +17,16 @@ class NamespacedRedis
     @redis.srem namespaced(key), member
   end
 
+  def spop(key)
+    @redis.spop namespaced(key)
+  end
+
   def smembers(key)
     @redis.smembers namespaced(key)
+  end
+  
+  def scard(key)
+    @redis.scard namespaced(key)
   end  
   
   def get(key)
@@ -29,6 +37,10 @@ class NamespacedRedis
     @redis.incr namespaced(key)
   end
   
+  def decr(key)
+    @redis.decr namespaced(key)
+  end
+  
   def exist(key)
     @redis.exist namespaced(key)
   end

data/lib/redirect_error.rb

@@ -0,0 +1,2 @@
+class RedirectError < Exception
+end

data/lib/stats.rb

@@ -0,0 +1,32 @@
+require 'bundler/setup'
+require 'sinatra'
+require 'haml'
+
+class Stats < Sinatra::Base
+  
+  def self.update_statistics(statistics)
+    @@statistics = statistics
+  end
+  
+  def self.update_status(status)
+    @@status = status
+  end
+  
+  ap settings.root
+  set :views, settings.root + '/../views'
+  
+  get '/' do
+    @statistics = @@statistics
+    @status = @@status
+    haml :statistics
+  end
+  
+end
+
+thread = Thread.new do
+  Stats.run!
+  
+  ## we need to manually kill the main thread as sinatra traps the interrupts
+  Thread.main.kill
+end
+

data/spec/cobweb/cobweb_crawler_spec.rb

@@ -31,7 +31,7 @@ describe CobwebCrawler do
         
         crawler = CobwebCrawler.new({:cache => false, :quiet => false, :debug => false})
         
-        statistics = crawler.crawl("http://www.rockwellcottage.com/")
+        statistics = crawler.crawl("http://rockwellcottage.heroku.com/")
         
         ap statistics
         
@@ -43,8 +43,9 @@ describe CobwebCrawler do
 
         crawler = CobwebCrawler.new({:cache => false, :quiet => false, :debug => false})
         
-        statistics = crawler.crawl("http://www.rockwellcottage.com/") do |content|
+        statistics = crawler.crawl("http://www.rockwellcottage.com/") do |content, statistics|
           ap content[:url]
+          ap statistics[:average_length]
         end
         
         ap statistics

data/spec/cobweb/cobweb_spec.rb

@@ -1,6 +1,6 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
-describe CobWeb do
+describe Cobweb do
 
   before(:each) do
   
@@ -15,7 +15,7 @@ describe CobWeb do
                         "Server" => "gws",
                         "X-XSS-Protection" => "1; mode=block"}
 
-    @cobweb = CobWeb.new :quiet => true, :cache => nil
+    @cobweb = Cobweb.new :quiet => true, :cache => nil
   end  
 
   describe "with mock" do
@@ -40,7 +40,9 @@ describe CobWeb do
       @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)
       @mock_http_client.stub!(:read_timeout=).and_return(nil)      
       @mock_http_client.stub!(:open_timeout=).and_return(nil)      
-      @mock_http_client.stub!(:start).and_return(@mock_http_response)      
+      @mock_http_client.stub!(:start).and_return(@mock_http_response)
+      @mock_http_client.stub!(:address).and_return("www.baseurl.com")
+      @mock_http_client.stub!(:port).and_return("80 ")
       
       @mock_http_response.stub!(:code).and_return(200)
       @mock_http_response.stub!(:content_type).and_return("text/html")
@@ -69,7 +71,7 @@ describe CobWeb do
     end
     
     it "should generate a cobweb object" do
-      CobWeb.new.should be_an_instance_of CobWeb
+      Cobweb.new.should be_an_instance_of Cobweb
     end
     
     describe "get" do
@@ -130,7 +132,7 @@ describe CobWeb do
         
         before(:each) do
           @base_url = "http://redirect-me.com/redirect.html"
-          @cobweb = CobWeb.new(:follow_redirects => true, :quiet => true, :cache => nil)
+          @cobweb = Cobweb.new(:follow_redirects => true, :quiet => true, :cache => nil)
         end
         
         it "should flow through redirect" #do
@@ -155,8 +157,8 @@ describe CobWeb do
           
         #end
         it "should not follow with redirect disabled" do
-          @cobweb = CobWeb.new(:follow_redirects => false, :cache => nil)
-          @mock_http_client.should_receive(:start).and_return(@mock_http_redirect_response)
+          @cobweb = Cobweb.new(:follow_redirects => false, :cache => nil)
+          @mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
           
           content = @cobweb.get(@base_url)
           content[:url].should == "http://redirect-me.com/redirect.html"

data/views/statistics.haml

@@ -0,0 +1,71 @@
+%h1 Cobweb Statistics
+
+%h4= @status
+
+%table
+  %tr
+    %th Page Count
+    %th Asset Count
+    %th Redirect Count  
+  %tr
+    %td= @statistics[:page_count]
+    %td= @statistics[:asset_count]
+    %td= @statistics[:total_redirects]  
+
+%table
+  %tr
+    %th Total Page Size
+    %th Total Asset Size
+  %tr
+    %td= @statistics[:page_size]
+    %td= @statistics[:asset_size]
+
+%table
+  %tr
+    %th Crawled
+    %th Queued
+  %tr
+    %td= @statistics[:crawl_counter]
+    %td= @statistics[:queue_counter]
+
+
+%table
+  %tr
+    %th{:colspan => 2} Response Times
+  %tr
+    %th Average
+    %td= @statistics[:average_response_time]
+  %tr
+    %th Maximum
+    %td= @statistics[:maximum_response_time]
+  %tr
+    %th Minimum
+    %td= @statistics[:minimum_response_time]
+
+%table
+  %tr
+    %th{:colspan => 2} Content Sizes
+  %tr
+    %th Average
+    %td= @statistics[:average_length]
+  %tr
+    %th Maximum
+    %td= @statistics[:maximum_length]
+  %tr
+    %th Minimum
+    %td= @statistics[:minimum_length]
+  %tr
+    %th Total
+    %td= @statistics[:total_length]
+
+%table
+  - @statistics[:mime_counts].keys.each do |mime_type|
+    %tr
+      %td= mime_type
+      %td= @statistics[:mime_counts][mime_type]
+
+%table
+  - @statistics[:status_counts].keys.each do |status|
+    %tr
+      %td= status
+      %td= @statistics[:status_counts][status]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.17
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-10 00:00:00.000000000 Z
+date: 2012-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70125097329480 !ruby/object:Gem::Requirement
+  requirement: &70173021660540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70125097329480
+  version_requirements: *70173021660540
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70125097328760 !ruby/object:Gem::Requirement
+  requirement: &70173021659920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70125097328760
+  version_requirements: *70173021659920
 - !ruby/object:Gem::Dependency
   name: absolutize
-  requirement: &70125097328280 !ruby/object:Gem::Requirement
+  requirement: &70173021659420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70125097328280
+  version_requirements: *70173021659420
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70125097327660 !ruby/object:Gem::Requirement
+  requirement: &70173021658760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70125097327660
+  version_requirements: *70173021658760
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70125097327060 !ruby/object:Gem::Requirement
+  requirement: &70173021658080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70125097327060
+  version_requirements: *70173021658080
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70125097326400 !ruby/object:Gem::Requirement
+  requirement: &70173021657320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,40 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70125097326400
+  version_requirements: *70173021657320
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: &70173021654680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70173021654680
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: &70173021654060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70173021654060
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: &70173021653560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70173021653560
 description: 
 email: stewart@rockwellcottage.com
 executables: []
@@ -96,7 +129,11 @@ files:
 - lib/cobweb_process_job.rb
 - lib/content_link_parser.rb
 - lib/crawl_job.rb
+- lib/hash.rb
 - lib/namespaced_redis.rb
+- lib/redirect_error.rb
+- lib/stats.rb
+- views/statistics.haml
 - README.textile
 homepage: http://github.com/stewartmckee/cobweb
 licenses: []
@@ -118,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Web Crawler that uses resque background job engine to allow you to cluster