rredis 0.1.0

data/bin/rredis_server

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'rredis'
+require 'rredis/server'
+require 'vegas'
+
+Vegas::Runner.new(RReDisServer::Web, 'rredis_server')

data/config.ru

@@ -0,0 +1,5 @@
+require 'rredis'
+
+
+require 'rredis/server'
+run RReDisServer::Web

data/lib/lua/get.lua

@@ -0,0 +1,74 @@
+local key = "rrd_" .. KEYS[1]
+local function get_value(value) 
+  local n = string.find(value, '_');
+  if n then
+    return string.sub(value, n+1, -1)
+  else
+    return tonumber(value)
+  end
+end
+local function get_data(data, offset)
+  local values = {}
+  local timestamps = {}
+  for i, d in ipairs(data) do
+    if i % 2 == 0 then
+      table.insert(timestamps, get_value(d)-offset)
+    else
+      table.insert(values, get_value(d))
+    end
+  end
+  return {timestamps, values}
+end
+
+-- Load the config
+local config = cjson.decode(redis.call("get", key .. "_config"))
+-- If we do not have a config we can assume that we also have no data to return
+if not config then
+  return {{}, {}}
+end
+
+
+local start = tonumber(ARGV[1])
+local stop = tonumber(ARGV[2])
+local timespan = stop-start
+
+local method
+if ARGV[3] == "" then
+  method = 'average'
+else
+  method = ARGV[3]
+end
+
+local higher_key = key..'_'..config["steps"]
+
+local oldest = redis.call("ZRANGE", higher_key, 0, 0, 'WITHSCORES')
+if not oldest then
+  return {{}, {}}
+end
+
+local oldest = tonumber(oldest[2])
+
+if timespan <= config.steps*config.rows and timespan/config.steps < 650 then
+  local data = redis.call("ZRANGEBYSCORE", higher_key, start, stop, 'WITHSCORES' )
+  return get_data(data, 0)
+end
+
+if config["rra"] then
+  local higher = config
+  local rra_count = table.getn(config.rra)
+  local rra_key, oldest
+  for i, rra in ipairs(config["rra"]) do
+    -- Get all entries from the higher precision bucket
+    rra_key = key..'_'..rra["steps"]..'_'..method
+    oldest = redis.call("ZRANGE", rra_key, 0, 0, 'WITHSCORES')
+    if oldest == {} then
+      return {{}, {}}
+    end
+
+    oldest = tonumber(oldest[2])
+    if (timespan <= rra.steps*rra.rows and timespan/rra.steps < 650) or i == rra_count then
+      local data = redis.call("ZRANGEBYSCORE", rra_key, start, stop, 'WITHSCORES' )
+      return get_data(data, rra.steps/2)
+    end
+  end
+end

data/lib/lua/store.lua

@@ -0,0 +1,170 @@
+local config, oldest
+local key = "rrd_" .. KEYS[1]
+-- Check if there is a config for this metric
+if redis.call("exists",key .. "_config") == 0 then
+  -- Create a config based on the default one
+  config = redis.call("get", "rrd_default_config")
+  redis.call("set", key .. "_config", config)
+  redis.call("sadd", "rrd_metrics_set", KEYS[1])
+end
+
+-- Load the config
+config = cjson.decode(redis.call("get", key .. "_config"))
+
+local timestamp = tonumber(ARGV[2])
+
+-- If steps are defined for the native resolution, we will round the timestamp 
+local higher_key = key..'_'..config["steps"]
+if (timestamp % config["steps"]) / config["steps"] <= 0.5 then
+  timestamp = math.floor(timestamp - (timestamp % config["steps"]))
+else
+  timestamp = math.floor(timestamp - (timestamp % config["steps"])) + config["steps"]
+end
+
+-- Get the amount of entries in this bucket
+local count = redis.call("ZCARD", higher_key)
+
+-- We need to make sure that to old entries are not added to the bucked
+if count+1 == config["rows"] then
+  oldest = tonumber(redis.call("ZRANGE", higher_key, 0, 0, 'WITHSCORES')[2])
+  if timestamp < oldest then
+    -- We cannot add older entries 
+    return false
+  end
+end
+
+-- We may be updating an old entry, which we want to delete first
+redis.call("ZREMRANGEBYSCORE", higher_key, timestamp, timestamp)
+
+-- Add the new entry to the bucked
+redis.call("ZADD", higher_key, timestamp, timestamp..'_'..ARGV[1])
+
+if count+1 > config["rows"] then 
+  -- We have too many entries in this bucket - remove the oldest
+  redis.call("ZREMRANGEBYRANK", higher_key, 0, (count-config.rows))
+end
+
+local get_value = function(value) return tonumber(string.sub(value, string.find(value, '_')+1, -1)) end
+
+if config["rra"] then
+  local higher = config
+  local lower_start, higher_ts, value, rra_key, last_rra, n
+
+  for j, rra in ipairs(config["rra"]) do
+    -- Calculate the timestamp for the aggregation
+    local steps = rra.steps
+    local rest = (timestamp % rra.steps) 
+    if rest == 0 then
+      lower_start = timestamp - rra.steps + higher["steps"]
+      higher_ts = timestamp
+    else
+      lower_start = timestamp - rest + higher["steps"]
+      higher_ts = timestamp - rest + rra.steps
+    end
+
+    if config["aggregations"] then
+      local a = {}
+      -- Shortcut, if we are in the first rra we can calculate the aggregations faster
+      if higher.steps == config.steps then 
+        -- Get all entries from the higher precision bucket
+        local data = redis.call(         "ZRANGEBYSCORE", higher_key, lower_start, lower_start+rra.steps)
+        -- If steps are defined for the native resolution, only proceed if we have enough entries
+        if not (table.getn(data) > (rra["steps"]/higher["steps"]*rra["xff"])) then
+          return false
+        end
+
+        a.sum = 0
+        a.min = 4503599627370496
+        a.max = -4503599627370496
+        for i, value in ipairs(data) do
+          value = get_value(value)
+          a.sum = a.sum + value
+          if value < a.min then
+            a.min = value
+          end
+          if value > a.max then
+            a.max = value
+          end
+        end
+        a.avg = a.sum / table.getn(data)
+      else
+        -- For every other rra we need to fetch the matching data
+        for i, method in ipairs(config["aggregations"]) do
+          -- Get all entries from the higher precision bucket
+          local data = redis.call(         "ZRANGEBYSCORE", higher_key..'_'..method, lower_start, lower_start+rra.steps)
+          -- If steps are defined for the native resolution, only proceed if we have enough entries
+          if not (table.getn(data) > (rra["steps"]/higher["steps"]*rra["xff"])) then
+            return false
+          end
+
+          if method == "average" then
+            a.sum = 0
+
+            for i, value in ipairs(data) do
+              a.sum = a.sum + get_value(value)
+            end
+            a.avg = a.sum / table.getn(data)
+          elseif method == "sum" then
+            a.sum = 0
+
+            for i, v in ipairs(data) do
+              a.sum = a.sum + get_value(v)
+            end
+          elseif method == "min" then
+            a.min = 4503599627370496
+            for i, value in ipairs(data) do
+              n = get_value(value)
+              if n < a.min then
+                a.min = n 
+              end
+            end
+          elseif method == "max" then
+            a.max = -4503599627370496
+            for i, value in ipairs(data) do
+              n = get_value(value)
+              if n > a.max then
+                a.max = n
+              end
+            end
+          else
+            redis.log(redis.LOG_ERROR, "Not implemented")
+          end
+        end
+      end
+
+      -- Update the buckets
+      for i, method in ipairs(config["aggregations"]) do
+        rra_key = key..'_'..rra["steps"]..'_'..method
+        -- We may be updating an old entry, which we want to delete first
+        redis.call("ZREMRANGEBYSCORE", rra_key, higher_ts, higher_ts)
+        -- Get the amount of entries in this bucket
+        count = redis.call("ZCARD", rra_key)
+
+        if count > higher["rows"] then 
+          -- We have too many entries in this bucket - remove the oldest
+          oldest = redis.call("ZRANGE", rra_key, 0, 0)
+          redis.call("ZREM", rra_key, oldest[1])
+        end
+        
+        -- Add the new entry to the bucked
+        if method == "average" then
+          redis.call("ZADD", rra_key, higher_ts, higher_ts..'_'..a.avg)
+        elseif method == "sum" then
+          redis.call("ZADD", rra_key, higher_ts, higher_ts..'_'..a.sum)
+        elseif method == "min" then
+          redis.call("ZADD", rra_key, higher_ts, higher_ts..'_'..a.min)
+        elseif method == "max" then
+          redis.call("ZADD", rra_key, higher_ts, higher_ts..'_'..a.max)
+        end
+        
+        last_rra = rra
+      end
+        
+      -- Set the higher precision bucket to the current rra
+      higher = last_rra
+      higher_key = key..'_'..higher["steps"]
+    end
+  end
+end
+
+return true

data/lib/rredis.rb

@@ -0,0 +1,61 @@
+require 'json'
+require 'redis'
+
+class RReDis
+  VERSION = '0.1.0'
+  attr_reader :default_config
+  
+  def initialize(options = {})
+    @r = Redis.new
+    @default_config = {:steps=>10, :rows=>8640, 
+                       :aggregations=>["average", "min", "max"], 
+                       :rra => [ {:steps=>60, :rows=>10080, :xff=>0.5},
+                                 {:steps=>900, :rows=>2976, :xff=>0.5},
+                                 {:steps=>3600, :rows=>8760, :xff=>0.5}]}
+    self.default_config = @default_config
+
+
+    @script_cache = {}
+    @sha_cache = {}
+    Dir.glob(File.join(File.dirname(__FILE__), '/lua/*.lua')).each do |file|
+      name = File.basename(file, File.extname(file))
+      @script_cache[name.to_sym] = File.open(file).read
+      @sha_cache[name.to_sym] = @r.script(:load, @script_cache[name.to_sym])
+    end
+    
+  end
+
+  def default_config=(config)
+    self.config('default', config, false)
+  end
+
+  def config(metric, config, set_metric=true)
+    config[:rra] = config[:rra].sort {|a,b| a[:steps] <=> b[:steps] } if config[:rra]
+    @r.set("rrd_#{metric}_config", JSON.dump(config))
+    @r.sadd "rrd_metrics_set", metric if set_metric
+  end
+
+  def store(metric, timestamp, value=nil)
+    if value.nil?
+      value = timestamp
+      timestamp = Time.now
+    end
+    @r.evalsha(@sha_cache[:store], :keys => [metric], :argv => [value, timestamp.to_f])
+  end
+
+  def get(metric, start, stop, method = nil)
+    resp = @r.evalsha(@sha_cache[:get], :keys => [metric], :argv => [start.to_i, stop.to_i, method])
+    if resp
+      resp[1].collect! { |x| x.to_f} 
+      resp
+    else
+      [[],[]]
+    end
+  end
+
+  def pipelined(&block)
+    @r.pipelined do
+      yield self
+    end
+  end
+end

data/lib/rredis/server.rb

@@ -0,0 +1,63 @@
+require 'sinatra/base'
+require 'slim'
+require 'sprockets'
+
+module RReDisServer
+  class SprocketsMiddleware
+    def initialize(app, options={})
+      @app = app
+      @root = options[:root]
+      path   =  options[:path] || 'assets'
+      @matcher = /^\/#{path}\/*/
+      @environment = ::Sprockets::Environment.new(@root)
+      @environment.append_path 'assets/javascripts'
+      @environment.append_path 'assets/javascripts/vendor'
+      @environment.append_path 'assets/stylesheets'
+      @environment.append_path 'assets/stylesheets/vendor'
+      @environment.append_path 'assets/images'
+    end
+
+    def call(env)
+      return [301, { 'Location' => "#{env['SCRIPT_NAME']}/" }, []] if env['SCRIPT_NAME'] == env['REQUEST_PATH']
+
+      return @app.call(env) unless @matcher =~ env["PATH_INFO"]
+      env['PATH_INFO'].sub!(@matcher,'')
+      @environment.call(env)
+    end
+  end
+
+  class Web < Sinatra::Base
+    dir = File.expand_path(File.dirname(__FILE__) + "/web")
+    set :views,  "#{dir}/views"
+    set :root, "#{dir}/assets"
+    set :slim, :pretty => true
+    use SprocketsMiddleware, :root => dir
+
+    
+    helpers do
+      def root_path
+        "#{env['SCRIPT_NAME']}/"
+      end
+    end
+
+    def initialize
+      super
+      @r = Redis.new
+      @rrd = RReDis.new
+    end
+
+    get "/" do
+      @metrics = @r.smembers("rrd_metrics_set")
+      slim :index
+    end
+
+    get "/get" do
+      data = {}
+      params['aggregations'].split(',').each do |method|
+        data[method] = @rrd.get(params['metric'], Time.now-params['timespan'].to_i, Time.now, method)
+      end
+      JSON.dump(data)
+    end
+  end
+
+end

data/lib/rredis/web/assets/javascripts/application.js

@@ -0,0 +1,3 @@
+//= require vendor/jquery
+//= require vendor/bootstrap
+//= require_tree .

data/lib/rredis/web/assets/javascripts/main.js

@@ -0,0 +1,97 @@
+function RReDis() {
+  that = this;
+  this.charts = {};
+  this.config = [{name: "10 minutes", steps: 60*10}, {name: "1 hour", steps: 3600}, {name: "12 hours", steps: 3600*12}, {name: "1 day", steps: 3600*24},{name: "30 days", steps: 3600*24*31}];
+  $("#metrics").change(function(value) {
+    that.metric = $(this).val();
+    that.createGraphs(that.metric);
+  });
+  this.createGraphs = function(metric) {
+    var now = new Date().getTime()/1000;
+    $("#charts").html("")
+    var d = new Date();
+    for(var i = 0; i < this.config.length; i++) {
+      var config = this.config[i];
+      $("#charts").append('<div id="'+metric+config.steps+'" class="chart"></div>');
+      this.charts[metric+config.steps] = new Highcharts.Chart({
+                                              chart: {
+                                                  renderTo: metric+config.steps,
+                                                  defaultSeriesType: 'line',
+                                                  events: {
+                                                      load: this.requestData(metric, config)
+                                                  },
+                                                  zoomType: 'xy'
+
+                                              },
+                                              tooltip: {
+                                                  xDateFormat: '%Y-%m-%d %H:%M:%S',
+                                                  shared: true
+                                              },
+
+                                              title: {
+                                                  text: config.name
+                                              },
+                                              xAxis: {
+                                                  type: 'datetime',
+                                                  tickPixelInterval : 50,
+                                                  labels: {
+                                                    rotation: -45,
+                                                    align: 'right',
+                                                  },
+                                                  maxZoom: 20 * 1000,
+                                                  max: d.getTime(),
+                                                  min: d.getTime()-(config.steps*1000)
+                                              },
+                                              yAxis: {
+                                                  minPadding: 0.2,
+                                                  maxPadding: 0.2,
+                                                  title: {
+                                                      text: 'Value',
+                                                      margin: 80
+                                                  }
+                                              },
+                                              plotOptions: {
+                                                line: {
+                                                  lineWidth: 1,
+                                                  marker: {
+                                                    enabled: false,
+                                                    states: {
+                                                      hover: {
+                                                        enabled: true,
+                                                        radius: 5
+                                                      }
+                                                    }
+                                                  },
+                                                }
+                                              },
+                                              series: [{
+                                                  name: metric,
+                                                  data: [ ]
+                                              },
+                                              { name: metric + "min",
+                                                data: []},
+                                              { name: metric + "max",
+                                                data: []}]
+                                          });
+    } 
+  };
+  this.requestData = function(metric, config) {
+    var that = this;
+    $.getJSON('get', {metric: metric, timespan: config.steps, aggregations: "average,min,max"},
+      function(data) {
+        var i = 0;
+        
+        for(method in data) {
+          var items = [];
+          $.each(data[method][0], function(key, val) {
+            items.push([val*1000, data[method][1][key]]);
+          });
+          
+          that.charts[metric+config.steps].series[i].setData (items, true, false);
+          i++;
+        }
+
+      }       
+    );
+  };
+}