cobweb 0.0.38 → 0.0.39

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.38
+h1. Cobweb v0.0.39
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -20,7 +20,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.38"
+    "0.0.39"
   end
   
   def method_missing(method_sym, *arguments, &block)
@@ -67,6 +67,8 @@ class Cobweb
     @redis.set("crawl-counter", 0)
     @redis.set("queue-counter", 1)
     
+    @stats = Stats.new(request)
+    @stats.start_crawl(request)
     
     # add internal_urls into redis
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}

data/lib/cobweb_crawler.rb

@@ -11,9 +11,18 @@ class CobwebCrawler
     @statistic = {}
     
     @options[:redis_options] = {:host => "127.0.0.1"} unless @options.has_key? :redis_options
-    crawl_id = Digest::MD5.hexdigest(DateTime.now.inspect.to_s)
+    @crawl_id = Digest::MD5.hexdigest(DateTime.now.inspect.to_s)
+    @options[:crawl_id] = @crawl_id
     
-    @redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{crawl_id}")
+    @redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{@crawl_id}")
+    @options[:internal_urls] = [] if @options[:internal_urls].nil?
+    @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
+    @debug = @options[:debug]
+    
+    @stats = Stats.new(@options.merge(:crawl_id => @crawl_id))
+    if @options[:web_statistics]
+      Server.start
+    end
     
     @cobweb = Cobweb.new(@options)
   end
@@ -21,124 +30,113 @@ class CobwebCrawler
   def crawl(base_url, crawl_options = {}, &block)
     @options[:base_url] = base_url unless @options.has_key? :base_url
     
+    @options[:internal_urls] << base_url if @options[:internal_urls].empty?
+    @redis.sadd("internal_urls", base_url) if @options[:internal_urls].empty?
+    
     @crawl_options = crawl_options
     
+    puts "http://localhost:4567/statistics/#{@crawl_id}"
+    puts ""
+    
     @redis.sadd "queued", base_url
     crawl_counter = @redis.scard("crawled").to_i
     queue_counter = @redis.scard("queued").to_i
 
-    while queue_counter>0 && (@options[:crawl_limit].to_i == 0 || @options[:crawl_limit].to_i > crawl_counter)      
-      thread = Thread.new do
-
-        url = @redis.spop "queued"
-        crawl_counter = @redis.scard("crawled").to_i
-        queue_counter = @redis.scard("queued").to_i
+    begin
+      @stats.start_crawl(@options)
+      while queue_counter>0 && (@options[:crawl_limit].to_i == 0 || @options[:crawl_limit].to_i > crawl_counter)      
+        thread = Thread.new do
         
-        @options[:url] = url
-        unless @redis.sismember("crawled", url.to_s)
-          begin
-            Stats.update_status("Requesting #{url}...")
-            content = @cobweb.get(url)
-            Stats.update_status("Processing #{url}...")
-
-            if @statistic[:average_response_time].nil?
-              @statistic[:average_response_time] = content[:response_time].to_f
-            else
-              @statistic[:average_response_time] = (((@statistic[:average_response_time] * crawl_counter) + content[:response_time].to_f) / (crawl_counter + 1))
-            end
-      
-            @statistic[:maximum_response_time] = content[:response_time] if @statistic[:maximum_response_time].nil? || @statistic[:maximum_response_time] < content[:response_time]
-            @statistic[:minimum_response_time] = content[:response_time] if @statistic[:minimum_response_time].nil? || @statistic[:minimum_response_time] > content[:response_time]
-      
-            if @statistic[:average_length]
-              @statistic[:average_length] = (((@statistic[:average_length].to_i*crawl_counter) + content[:length].to_i) / (crawl_counter + 1))
-            else
-              @statistic[:average_length] = content[:length].to_i
-            end
-      
-            @statistic[:maximum_length] = content[:length].to_i if @statistic[:maximum_length].nil? || content[:length].to_i > @statistic[:maximum_length].to_i
-            @statistic[:minimum_length] = content[:length].to_i if @statistic[:minimum_length].nil? || content[:length].to_i < @statistic[:minimum_length].to_i
-            @statistic[:total_length] = @statistic[:total_length].to_i + content[:length].to_i
+          url = @redis.spop "queued"
+          crawl_counter = @redis.scard("crawled").to_i
+          queue_counter = @redis.scard("queued").to_i
+        
+          @options[:url] = url
+          unless @redis.sismember("crawled", url.to_s)
+            begin
+              @stats.update_status("Requesting #{url}...")
+              content = @cobweb.get(url)
+              @stats.update_status("Processing #{url}...")
 
-            if content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
-              @statistic[:page_count] = @statistic[:page_count].to_i + 1
-              @statistic[:page_size] = @statistic[:page_size].to_i + content[:length].to_i
-            else
-              @statistic[:asset_count] = @statistic[:asset_count].to_i + 1
-              @statistic[:asset_size] = @statistic[:asset_size].to_i + content[:length].to_i
-            end
-            
-            @statistic[:total_redirects] = 0 if @statistic[:total_redirects].nil?
-            @statistic[:total_redirects] += content[:redirect_through].count unless content[:redirect_through].nil?
+              @redis.sadd "crawled", url.to_s
+              @redis.incr "crawl-counter" 
             
-            @statistic[:crawl_counter] = crawl_counter
-            @statistic[:queue_counter] = queue_counter
+              internal_links = all_links_from_content(content).map{|link| link.to_s}
+
+              # reject the link if we've crawled it or queued it
+              internal_links.reject!{|link| @redis.sismember("crawled", link)}
+              internal_links.reject!{|link| @redis.sismember("queued", link)}
             
-            mime_counts = {}
-            if @statistic.has_key? :mime_counts
-              mime_counts = @statistic[:mime_counts]
-              if mime_counts.has_key? content[:mime_type]
-                mime_counts[content[:mime_type]] += 1
-              else
-                mime_counts[content[:mime_type]] = 1
-              end
-            else
-              mime_counts = {content[:mime_type] => 1}
-            end
-            @statistic[:mime_counts] = mime_counts
 
-            status_counts = {}
-          
-            if @statistic.has_key? :status_counts
-              status_counts = @statistic[:status_counts]
-              if status_counts.has_key? content[:status_code].to_i
-                status_counts[content[:status_code].to_i] += 1
-              else
-                status_counts[content[:status_code].to_i] = 1
-              end
-            else
-              status_counts = {content[:status_code].to_i => 1}
-            end
-            @statistic[:status_counts] = status_counts
+              # select the link if its internal
+              internal_links.select!{|link| internal_link?(link)}
 
-            @redis.sadd "crawled", url.to_s
-            @redis.incr "crawl-counter" 
-            
-            content[:links].keys.map{|key| content[:links][key]}.flatten.each do |content_link|
-              link = content_link.to_s
-              unless @redis.sismember("crawled", link)
-                puts "Checking if #{link} matches #{@options[:base_url]} as internal?" if @options[:debug]
-                if link.to_s.match(Regexp.new("^#{@options[:base_url]}"))
-                  puts "Matched as #{link} as internal" if @options[:debug]
-                  unless @redis.sismember("crawled", link) || @redis.sismember("queued", link)
-                    puts "Added #{link.to_s} to queue" if @options[:debug]
-                    @redis.sadd "queued", link
-                    crawl_counter = @redis.scard("crawled").to_i
-                    queue_counter = @redis.scard("queued").to_i
-                  end
-                end
+              internal_links.each do |link|
+                puts "Added #{link.to_s} to queue" if @debug
+                @redis.sadd "queued", link
               end
-            end
             
-            crawl_counter = @redis.scard("crawled").to_i
-            queue_counter = @redis.scard("queued").to_i
-            Stats.update_statistics(@statistic)
-            Stats.update_status("Completed #{url}.")
-            puts "Crawled: #{crawl_counter.to_i} Limit: #{@options[:crawl_limit].to_i} Queued: #{queue_counter.to_i}" if @options[:debug] 
+              crawl_counter = @redis.scard("crawled").to_i
+              queue_counter = @redis.scard("queued").to_i
+
+              @stats.update_statistics(content)
+              @stats.update_status("Completed #{url}.")
+              puts "Crawled: #{crawl_counter.to_i} Limit: #{@options[:crawl_limit].to_i} Queued: #{queue_counter.to_i}" if @debug 
        
-            yield content, @statistic if block_given?
+              yield content, @statistic if block_given?
 
-          rescue => e
-            puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
-            ap e
-            ap e.backtrace
+            rescue => e
+              puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
+              ap e
+              ap e.backtrace
+            end
+          else
+            puts "Already crawled #{@options[:url]}" if @debug
           end
-        else
-          puts "Already crawled #{@options[:url]}" if @options[:debug]
         end
+        thread.join
       end
-      thread.join
+    ensure
+      @stats.end_crawl(@options)
     end
     @statistic
   end
+  
+  
+  def internal_link?(link)
+    puts "Checking internal link for: #{link}" if @debug
+    valid_link = true
+    internal_patterns.map{|pattern| Regexp.new("^#{pattern.gsub("*", ".*?")}")}.each do |pattern|
+      puts "Matching against #{pattern.source}" if @debug
+      if link.match(pattern)
+        puts "Matched as internal" if @debug
+        return true
+      end
+    end
+    puts "Didn't match any pattern so marked as not internal" if @debug
+    false
+  end
+  
+  def internal_patterns
+    @internal_patterns ||= @redis.smembers("internal_urls")
+  end
+
+  def all_links_from_content(content)
+    links = content[:links].keys.map{|key| content[:links][key]}.flatten
+    links.reject!{|link| link.starts_with?("javascript:")}
+    links = links.map{|link| UriHelper.join_no_fragment(content[:url], link) }
+    links.select!{|link| link.scheme == "http" || link.scheme == "https"}
+    links.uniq
+    links
+  end
+end
+
+class String
+  def starts_with?(val)
+    if self.length >= val.length
+      self[0..val.length-1] == val
+    else
+      false
+    end
+  end
 end

data/lib/crawl_job.rb

@@ -14,6 +14,7 @@ class CrawlJob
     
     content_request[:redis_options] = {} unless content_request.has_key? :redis_options
     @redis = NamespacedRedis.new(content_request[:redis_options], "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
+    @stats = Stats.new(content_request)
     
     @debug = content_request[:debug]
     
@@ -32,7 +33,7 @@ class CrawlJob
         content = Cobweb.new(content_request).get(content_request[:url], content_request)
         
         ## update statistics
-        Stats.set_statistics_in_redis(@redis, content)
+        @stats.update_statistics(content)
         
         # set the base url if this is the first page
         set_base_url @redis, content, content_request
@@ -81,9 +82,7 @@ class CrawlJob
 
   def self.finished(content_request)
     # finished
-    
-    Stats.set_totals
-
+    @stats.end_crawl(content_request)
     Resque.enqueue(const_get(content_request[:crawl_finished_queue]), Stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))                
   end
 

data/lib/server.rb

@@ -0,0 +1,98 @@
+require 'sinatra'
+require 'haml'
+
+class Server < Sinatra::Base
+
+  set :views, settings.root + '/../views'
+  puts "#{settings.root}/../public"
+  set :public_folder, settings.root + '/../public'
+  enable :static
+  
+  get '/' do
+    @full_redis = Redis.new
+    
+    @colors = ["#00366f", "#006ba0", "#3F0BDB", "#396CB3"]
+    
+    @crawls = []
+    @full_redis.smembers("cobweb_crawls").each do |crawl_id|
+      redis = NamespacedRedis.new({}, "cobweb-#{crawl_id}")
+      stats = HashUtil.deep_symbolize_keys({
+        :crawl_details => redis.hgetall("crawl_details"), 
+        :statistics => redis.hgetall("statistics"),
+        :minute_totals => redis.hgetall("minute_totals")
+        })
+      @crawls << stats
+    end
+    
+    haml :home
+  end
+  
+  get '/statistics/:crawl_id' do
+    redis = NamespacedRedis.new({}, "cobweb-#{params[:crawl_id]}")
+    
+    @statistics = HashUtil.deep_symbolize_keys(redis.hgetall("statistics"))
+    if @statistics[:status_counts].nil?
+      @statistics[:status_counts]
+    else
+      @statistics[:status_counts] = JSON.parse(@statistics[:status_counts])
+    end
+    if @statistics[:mime_counts].nil?
+      @statistics[:mime_counts]
+    else
+      @statistics[:mime_counts] = JSON.parse(@statistics[:mime_counts])
+    end
+    @crawl = {
+        :statistics => @statistics,
+        :crawl_details => HashUtil.deep_symbolize_keys(redis.hgetall("crawl_details")), 
+        :minute_totals => HashUtil.deep_symbolize_keys(redis.hgetall("minute_totals")),
+        :status_200_count => HashUtil.deep_symbolize_keys(redis.hgetall("status_200_count")),
+        :status_400_count => HashUtil.deep_symbolize_keys(redis.hgetall("status_400_count")),
+        :status_500_count => HashUtil.deep_symbolize_keys(redis.hgetall("status_500_count")),
+        :mime_text_count => HashUtil.deep_symbolize_keys(redis.hgetall("mime_text_count")),
+        :mime_image_count => HashUtil.deep_symbolize_keys(redis.hgetall("mime_image_count")),
+        :mime_application_count => HashUtil.deep_symbolize_keys(redis.hgetall("mime_application_count")),
+        :pages_count => HashUtil.deep_symbolize_keys(redis.hgetall("pages_count")),
+        :assets_count => HashUtil.deep_symbolize_keys(redis.hgetall("assets_count"))
+    }
+    ap @crawl
+    haml :statistics
+  end
+  
+  def self.start
+    unless Server.running?
+      thread = Thread.new do
+        puts "Starting Sinatra"
+        Server.run!
+        puts "Stopping crawl..."
+        ## we need to manually kill the main thread as sinatra traps the interrupts
+        Thread.main.kill
+      end
+    end    
+  end  
+  
+end
+
+class HashUtil
+  def self.deep_symbolize_keys(hash)
+    hash.keys.each do |key|
+      value = hash[key]
+      hash.delete(key)
+      hash[key.to_sym] = value
+      if hash[key.to_sym].instance_of? Hash
+        hash[key.to_sym] = HashUtil.deep_symbolize_keys(hash[key.to_sym])
+      end
+    end
+    hash
+  end
+end
+
+class Numeric
+  def to_human
+    units = %w{B KB MB GB TB}
+    ap self
+    e = 0
+    e = (Math.log(self)/Math.log(1024)).floor unless self==0
+    s = "%.3f" % (to_f / 1024**e)
+    s.sub(/\.?0*$/, units[e])
+  end
+end

data/lib/stats.rb

@@ -1,63 +1,68 @@
-require 'sinatra'
-require 'haml'
-
-class Stats < Sinatra::Base
-  
-  def self.update_statistics(statistics)
-    @@statistics = statistics
-    @@statistics
-  end
+class Stats
   
-  def self.get_statistics
-    @@statistics ||= {}
+  def initialize(options)
+    @full_redis = Redis.new(options[:redis_options])
+    @redis = NamespacedRedis.new(options[:redis_options], "cobweb-#{options[:crawl_id]}")
   end
   
-  def self.update_status(status)
-    @@status = status
+  def start_crawl(options)
+    @full_redis.sadd "cobweb_crawls", options[:crawl_id]
+    options.keys.each do |key|
+      @redis.hset "crawl_details", key, options[key]
+    end
+    @redis.hset "statistics", "current_status", "Crawl Starting..."
   end
   
-  def self.set_totals
-    stats = @redis.hgetall "statistics"
-    stats[:total_pages] = @redis.get("total_pages").to_i
-    stats[:total_assets] = @redis.get("total_assets").to_i
-    stats[:crawl_counter] = @crawl_counter
-    stats[:queue_counter] = @queue_counter
-    stats[:crawled] = @redis.smembers "crawled"
-
-    Stats.update_statistics(stats)
+  def end_crawl(options)
+    @full_redis.srem "cobweb_crawls", options[:crawl_id]
+    @redis.hset "statistics", "current_status", "Crawl Stopped"
+    @redis.del "crawl_details"
   end
   
-  def self.set_statistics_in_redis(redis, content)
-    
-    @redis = redis
+  def update_statistics(content)
     
-    crawl_counter = redis.get("crawl-counter").to_i
-    queue_counter = redis.get("queue-counter").to_i
+    crawl_counter = @redis.scard("crawled").to_i
+    queue_counter = @redis.scard("queued").to_i
     
-    if redis.hexists "statistics", "average_response_time"
-      redis.hset("statistics", "average_response_time", (((redis.hget("statistics", "average_response_time").to_f*crawl_counter) + content[:response_time].to_f) / (crawl_counter + 1)))
+    if @redis.hexists "statistics", "average_response_time"
+      @redis.hset("statistics", "average_response_time", (((@redis.hget("statistics", "average_response_time").to_f*crawl_counter) + content[:response_time].to_f) / (crawl_counter + 1)))
     else
-      redis.hset("statistics", "average_response_time", content[:response_time].to_f)
+      @redis.hset("statistics", "average_response_time", content[:response_time].to_f)
     end
-    redis.hset "statistics", "maximum_response_time", content[:response_time].to_f if redis.hget("statistics", "maximum_response_time").nil? or content[:response_time].to_f > redis.hget("statistics", "maximum_response_time").to_f
-    redis.hset "statistics", "minimum_response_time", content[:response_time].to_f if redis.hget("statistics", "minimum_response_time").nil? or content[:response_time].to_f < redis.hget("statistics", "minimum_response_time").to_f
-    if redis.hexists "statistics", "average_length"
-      redis.hset("statistics", "average_length", (((redis.hget("statistics", "average_length").to_i*crawl_counter) + content[:length].to_i) / (crawl_counter + 1)))
+    @redis.hset "statistics", "maximum_response_time", content[:response_time].to_f if @redis.hget("statistics", "maximum_response_time").nil? or content[:response_time].to_f > @redis.hget("statistics", "maximum_response_time").to_f
+    @redis.hset "statistics", "minimum_response_time", content[:response_time].to_f if @redis.hget("statistics", "minimum_response_time").nil? or content[:response_time].to_f < @redis.hget("statistics", "minimum_response_time").to_f
+    if @redis.hexists "statistics", "average_length"
+      @redis.hset("statistics", "average_length", (((@redis.hget("statistics", "average_length").to_i*crawl_counter) + content[:length].to_i) / (crawl_counter + 1)))
     else
-      redis.hset("statistics", "average_length", content[:length].to_i)
+      @redis.hset("statistics", "average_length", content[:length].to_i)
     end
-    redis.hset "statistics", "maximum_length", content[:length].to_i if redis.hget("statistics", "maximum_length").nil? or content[:length].to_i > redis.hget("statistics", "maximum_length").to_i
-    redis.hset "statistics", "minimum_length", content[:length].to_i if redis.hget("statistics", "minimum_length").nil? or content[:length].to_i < redis.hget("statistics", "minimum_length").to_i
-
+    @redis.hset "statistics", "maximum_length", content[:length].to_i if @redis.hget("statistics", "maximum_length").nil? or content[:length].to_i > @redis.hget("statistics", "maximum_length").to_i
+    @redis.hset "statistics", "minimum_length", content[:length].to_i if @redis.hget("statistics", "minimum_length").nil? or content[:length].to_i < @redis.hget("statistics", "minimum_length").to_i
+  
+  
     if content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
-      redis.incr "total_pages"
+      @redis.hset "statistics", "page_count", @redis.hget("statistics", "page_count").to_i + 1
+      @redis.hset "statistics", "page_size", @redis.hget("statistics", "page_size").to_i + content[:length].to_i
+      increment_time_stat("pages_count")
     else
-      redis.incr "total_assets"
+      @redis.hset "statistics", "asset_count", @redis.hget("statistics", "asset_count").to_i + 1
+      @redis.hset "statistics", "asset_size", @redis.hget("statistics", "asset_size").to_i + content[:length].to_i
+      increment_time_stat("assets_count")
     end
+    
+    total_redirects = @redis.hget("statistics", "total_redirects").to_i
+    @redis.hset "statistics", "total_redirects", 0 if total_redirects.nil?
+    @redis.hset("statistics", "total_redirects", total_redirects += content[:redirect_through].count) unless content[:redirect_through].nil?
+
+    @redis.hset "statistics", "crawl_counter", crawl_counter
+    @redis.hset "statistics", "queue_counter", queue_counter
+    
+    total_length = @redis.hget("statistics", "total_length").to_i
+    @redis.hset "statistics", "total_length", total_length + content[:length].to_i
 
     mime_counts = {}
-    if redis.hexists "statistics", "mime_counts"
-      mime_counts = JSON.parse(redis.hget("statistics", "mime_counts"))
+    if @redis.hexists "statistics", "mime_counts"
+      mime_counts = JSON.parse(@redis.hget("statistics", "mime_counts"))
       if mime_counts.has_key? content[:mime_type]
         mime_counts[content[:mime_type]] += 1
       else
@@ -66,40 +71,112 @@ class Stats < Sinatra::Base
     else
       mime_counts = {content[:mime_type] => 1}
     end
-    redis.hset "statistics", "mime_counts", mime_counts.to_json
+    @redis.hset "statistics", "mime_counts", mime_counts.to_json
 
+    # record mime categories stats
+    if content[:mime_type].starts_with? "text"
+      increment_time_stat("mime_text_count")
+    elsif content[:mime_type].starts_with? "application"
+      increment_time_stat("mime_application_count")
+    elsif content[:mime_type].starts_with? "audio"
+      increment_time_stat("mime_audio_count")
+    elsif content[:mime_type].starts_with? "image"
+      increment_time_stat("mime_image_count")
+    elsif content[:mime_type].starts_with? "message"
+      increment_time_stat("mime_message_count")
+    elsif content[:mime_type].starts_with? "model"
+      increment_time_stat("mime_model_count")
+    elsif content[:mime_type].starts_with? "multipart"
+      increment_time_stat("mime_multipart_count")
+    elsif content[:mime_type].starts_with? "video"
+      increment_time_stat("mime_video_count")
+    end
+    
     status_counts = {}
-    if redis.hexists "statistics", "status_counts"
-      status_counts = JSON.parse(redis.hget("statistics", "status_counts"))
-      if status_counts.has_key? content[:status_code].to_i
-        status_counts[content[:status_code].to_i] += 1
+    if @redis.hexists "statistics", "status_counts"
+      status_counts = HashUtil.deep_symbolize_keys(JSON.parse(@redis.hget("statistics", "status_counts")))
+      status_code = content[:status_code].to_i.to_s.to_sym
+      if status_counts.has_key? status_code
+        status_counts[status_code] += 1
       else
-        status_counts[content[:status_code].to_i] = 1
-      end
+        status_counts[status_code] = 1
+      end      
     else
-      status_counts = {content[:status_code].to_i => 1}
+      status_counts = {status_code => 1}
+    end
+    
+    # record statistics by status type
+    if content[:status_code] >= 200 && content[:status_code] < 300
+      increment_time_stat("status_200_count")
+    elsif content[:status_code] >= 400 && content[:status_code] < 500
+      increment_time_stat("status|_400_count")
+    elsif content[:status_code] >= 500 && content[:status_code] < 600
+      increment_time_stat("status|_500_count")
+    end
+    
+    @redis.hset "statistics", "status_counts", status_counts.to_json
+    
+    
+    ## time based statistics
+    increment_time_stat("minute_totals", "minute", 60)    
+    
+    get_statistics
+  end
+  
+  def record_time_stat(stat_name, value, type="minute", duration=60)
+    key = DateTime.now.strftime("%Y-%m-%d %H:%M")
+    if type == "hour"
+      key = DateTime.now.strftime("%Y-%m-%d %H:00")
     end
-    redis.hset "statistics", "status_counts", status_counts.to_json
+    stat_value = @redis.hget(stat_name, key).to_i
+    stat_count = @redis.hget("#{stat_name}-count", key).to_i
     
-    @@statistics = @redis.hgetall "statistics"
+    if minute_count.nil?
+      @redis.hset stat_name, key, value
+      @redis.hset "#{stat_name}-count", key, 1
+    else
+      @redis.hset stat_name, key, ((stat_value*stat_count) + value) / (stat_count+1)
+      @redis.hset "#{stat_name}-count", key, stat_count+1
+    end
+  end
+  
+  def increment_time_stat(stat_name, type="minute", duration=60)
+    key = DateTime.now.strftime("%Y-%m-%d %H:%M")
+    if type == "hour"
+      key = DateTime.now.strftime("%Y-%m-%d %H:00")
+    end
+    minute_count = @redis.hget(stat_name, key).to_i
+    if minute_count.nil?
+      @redis.hset stat_name, key, 1
+    else
+      @redis.hset stat_name, key, minute_count + 1
+    end
+    #clear up older data
+    @redis.hgetall(stat_name).keys.each do |key|
+      if DateTime.parse(key) < DateTime.now-(duration/1440.0)
+        puts "Deleting #{stat_name} - #{key}"
+        @redis.hdel(stat_name, key)
+      end
+    end
+  end
+  
+  def get_statistics
+    @redis.hgetall "statistics"
   end
   
-  set :views, settings.root + '/../views'
+  def update_status(status)
+    @redis.hset "statistics", "current_status", status
+  end
   
-  get '/' do
-    @statistics = @@statistics
-    @status = @@status
-    haml :statistics
+  def get_status
+    @redis.hget "statistics", "current_status"
   end
   
-  def self.start
-    thread = Thread.new do
-      Stats.run!
-
-      ## we need to manually kill the main thread as sinatra traps the interrupts
-      Thread.main.kill
-    end    
+  def set_totals
+    stats = get_statistics
+    stats[:crawled] = @redis.smembers "crawled"
   end
+  
 end