rredis 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. data/.autotest +23 -0
  2. data/.gemtest +0 -0
  3. data/.rspec +2 -0
  4. data/CHANGELOG.md +3 -0
  5. data/Gemfile +18 -0
  6. data/Gemfile.lock +68 -0
  7. data/History.txt +6 -0
  8. data/Manifest.txt +40 -0
  9. data/README.md +143 -0
  10. data/Rakefile +34 -0
  11. data/benchmark/default.rb +62 -0
  12. data/benchmark/pipeline.rb +57 -0
  13. data/bin/rredis_server +7 -0
  14. data/config.ru +5 -0
  15. data/lib/lua/get.lua +74 -0
  16. data/lib/lua/store.lua +170 -0
  17. data/lib/rredis.rb +61 -0
  18. data/lib/rredis/server.rb +63 -0
  19. data/lib/rredis/web/assets/javascripts/application.js +3 -0
  20. data/lib/rredis/web/assets/javascripts/main.js +97 -0
  21. data/lib/rredis/web/assets/javascripts/vendor/bootstrap.js +1726 -0
  22. data/lib/rredis/web/assets/javascripts/vendor/bootstrap.min.js +6 -0
  23. data/lib/rredis/web/assets/javascripts/vendor/highcharts/highcharts.js +202 -0
  24. data/lib/rredis/web/assets/javascripts/vendor/highcharts/modules/canvas-tools.js +133 -0
  25. data/lib/rredis/web/assets/javascripts/vendor/highcharts/modules/exporting.js +23 -0
  26. data/lib/rredis/web/assets/javascripts/vendor/highcharts/themes/dark-blue.js +263 -0
  27. data/lib/rredis/web/assets/javascripts/vendor/highcharts/themes/dark-green.js +263 -0
  28. data/lib/rredis/web/assets/javascripts/vendor/highcharts/themes/gray.js +262 -0
  29. data/lib/rredis/web/assets/javascripts/vendor/highcharts/themes/grid.js +95 -0
  30. data/lib/rredis/web/assets/javascripts/vendor/highcharts/themes/skies.js +89 -0
  31. data/lib/rredis/web/assets/javascripts/vendor/jquery.js +4 -0
  32. data/lib/rredis/web/assets/stylesheets/application.css +10 -0
  33. data/lib/rredis/web/assets/stylesheets/vendor/bootstrap-responsive.css +686 -0
  34. data/lib/rredis/web/assets/stylesheets/vendor/bootstrap.css +3990 -0
  35. data/lib/rredis/web/images/glyphicons-halflings-white.png +0 -0
  36. data/lib/rredis/web/images/glyphicons-halflings.png +0 -0
  37. data/lib/rredis/web/views/index.slim +7 -0
  38. data/lib/rredis/web/views/layout.slim +26 -0
  39. data/spec/rredis/get_spec.rb +47 -0
  40. data/spec/rredis/store_spec.rb +159 -0
  41. data/spec/spec_helper.rb +21 -0
  42. metadata +124 -0
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rredis'
4
+ require 'rredis/server'
5
+ require 'vegas'
6
+
7
+ Vegas::Runner.new(RReDisServer::Web, 'rredis_server')
@@ -0,0 +1,5 @@
1
+ require 'rredis'
2
+
3
+
4
+ require 'rredis/server'
5
+ run RReDisServer::Web
@@ -0,0 +1,74 @@
1
+ local key = "rrd_" .. KEYS[1]
2
+ local function get_value(value)
3
+ local n = string.find(value, '_');
4
+ if n then
5
+ return string.sub(value, n+1, -1)
6
+ else
7
+ return tonumber(value)
8
+ end
9
+ end
10
+ local function get_data(data, offset)
11
+ local values = {}
12
+ local timestamps = {}
13
+ for i, d in ipairs(data) do
14
+ if i % 2 == 0 then
15
+ table.insert(timestamps, get_value(d)-offset)
16
+ else
17
+ table.insert(values, get_value(d))
18
+ end
19
+ end
20
+ return {timestamps, values}
21
+ end
22
+
23
+ -- Load the config
24
+ local config = cjson.decode(redis.call("get", key .. "_config"))
25
+ -- If we do not have a config we can assume that we also have no data to return
26
+ if not config then
27
+ return {{}, {}}
28
+ end
29
+
30
+
31
+ local start = tonumber(ARGV[1])
32
+ local stop = tonumber(ARGV[2])
33
+ local timespan = stop-start
34
+
35
+ local method
36
+ if ARGV[3] == "" then
37
+ method = 'average'
38
+ else
39
+ method = ARGV[3]
40
+ end
41
+
42
+ local higher_key = key..'_'..config["steps"]
43
+
44
+ local oldest = redis.call("ZRANGE", higher_key, 0, 0, 'WITHSCORES')
45
+ if not oldest then
46
+ return {{}, {}}
47
+ end
48
+
49
+ local oldest = tonumber(oldest[2])
50
+
51
+ if timespan <= config.steps*config.rows and timespan/config.steps < 650 then
52
+ local data = redis.call("ZRANGEBYSCORE", higher_key, start, stop, 'WITHSCORES' )
53
+ return get_data(data, 0)
54
+ end
55
+
56
+ if config["rra"] then
57
+ local higher = config
58
+ local rra_count = table.getn(config.rra)
59
+ local rra_key, oldest
60
+ for i, rra in ipairs(config["rra"]) do
61
+ -- Get all entries from the higher precision bucket
62
+ rra_key = key..'_'..rra["steps"]..'_'..method
63
+ oldest = redis.call("ZRANGE", rra_key, 0, 0, 'WITHSCORES')
64
+ if oldest == {} then
65
+ return {{}, {}}
66
+ end
67
+
68
+ oldest = tonumber(oldest[2])
69
+ if (timespan <= rra.steps*rra.rows and timespan/rra.steps < 650) or i == rra_count then
70
+ local data = redis.call("ZRANGEBYSCORE", rra_key, start, stop, 'WITHSCORES' )
71
+ return get_data(data, rra.steps/2)
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,170 @@
1
+ local config, oldest
2
+ local key = "rrd_" .. KEYS[1]
3
+ -- Check if there is a config for this metric
4
+ if redis.call("exists",key .. "_config") == 0 then
5
+ -- Create a config based on the default one
6
+ config = redis.call("get", "rrd_default_config")
7
+ redis.call("set", key .. "_config", config)
8
+ redis.call("sadd", "rrd_metrics_set", KEYS[1])
9
+ end
10
+
11
+ -- Load the config
12
+ config = cjson.decode(redis.call("get", key .. "_config"))
13
+
14
+ local timestamp = tonumber(ARGV[2])
15
+
16
+ -- If steps are defined for the native resolution, we will round the timestamp
17
+ local higher_key = key..'_'..config["steps"]
18
+ if (timestamp % config["steps"]) / config["steps"] <= 0.5 then
19
+ timestamp = math.floor(timestamp - (timestamp % config["steps"]))
20
+ else
21
+ timestamp = math.floor(timestamp - (timestamp % config["steps"])) + config["steps"]
22
+ end
23
+
24
+ -- Get the amount of entries in this bucket
25
+ local count = redis.call("ZCARD", higher_key)
26
+
27
+ -- We need to make sure that to old entries are not added to the bucked
28
+ if count+1 == config["rows"] then
29
+ oldest = tonumber(redis.call("ZRANGE", higher_key, 0, 0, 'WITHSCORES')[2])
30
+ if timestamp < oldest then
31
+ -- We cannot add older entries
32
+ return false
33
+ end
34
+ end
35
+
36
+ -- We may be updating an old entry, which we want to delete first
37
+ redis.call("ZREMRANGEBYSCORE", higher_key, timestamp, timestamp)
38
+
39
+ -- Add the new entry to the bucked
40
+ redis.call("ZADD", higher_key, timestamp, timestamp..'_'..ARGV[1])
41
+
42
+ if count+1 > config["rows"] then
43
+ -- We have too many entries in this bucket - remove the oldest
44
+ redis.call("ZREMRANGEBYRANK", higher_key, 0, (count-config.rows))
45
+ end
46
+
47
+ local get_value = function(value) return tonumber(string.sub(value, string.find(value, '_')+1, -1)) end
48
+
49
+ if config["rra"] then
50
+ local higher = config
51
+ local lower_start, higher_ts, value, rra_key, last_rra, n
52
+
53
+ for j, rra in ipairs(config["rra"]) do
54
+ -- Calculate the timestamp for the aggregation
55
+ local steps = rra.steps
56
+ local rest = (timestamp % rra.steps)
57
+ if rest == 0 then
58
+ lower_start = timestamp - rra.steps + higher["steps"]
59
+ higher_ts = timestamp
60
+ else
61
+ lower_start = timestamp - rest + higher["steps"]
62
+ higher_ts = timestamp - rest + rra.steps
63
+ end
64
+
65
+ if config["aggregations"] then
66
+ local a = {}
67
+ -- Shortcut, if we are in the first rra we can calculate the aggregations faster
68
+ if higher.steps == config.steps then
69
+ -- Get all entries from the higher precision bucket
70
+ local data = redis.call( "ZRANGEBYSCORE", higher_key, lower_start, lower_start+rra.steps)
71
+ -- If steps are defined for the native resolution, only proceed if we have enough entries
72
+ if not (table.getn(data) > (rra["steps"]/higher["steps"]*rra["xff"])) then
73
+ return false
74
+ end
75
+
76
+ a.sum = 0
77
+ a.min = 4503599627370496
78
+ a.max = -4503599627370496
79
+ for i, value in ipairs(data) do
80
+ value = get_value(value)
81
+ a.sum = a.sum + value
82
+ if value < a.min then
83
+ a.min = value
84
+ end
85
+ if value > a.max then
86
+ a.max = value
87
+ end
88
+ end
89
+ a.avg = a.sum / table.getn(data)
90
+ else
91
+ -- For every other rra we need to fetch the matching data
92
+ for i, method in ipairs(config["aggregations"]) do
93
+ -- Get all entries from the higher precision bucket
94
+ local data = redis.call( "ZRANGEBYSCORE", higher_key..'_'..method, lower_start, lower_start+rra.steps)
95
+ -- If steps are defined for the native resolution, only proceed if we have enough entries
96
+ if not (table.getn(data) > (rra["steps"]/higher["steps"]*rra["xff"])) then
97
+ return false
98
+ end
99
+
100
+ if method == "average" then
101
+ a.sum = 0
102
+
103
+ for i, value in ipairs(data) do
104
+ a.sum = a.sum + get_value(value)
105
+ end
106
+ a.avg = a.sum / table.getn(data)
107
+ elseif method == "sum" then
108
+ a.sum = 0
109
+
110
+ for i, v in ipairs(data) do
111
+ a.sum = a.sum + get_value(v)
112
+ end
113
+ elseif method == "min" then
114
+ a.min = 4503599627370496
115
+ for i, value in ipairs(data) do
116
+ n = get_value(value)
117
+ if n < a.min then
118
+ a.min = n
119
+ end
120
+ end
121
+ elseif method == "max" then
122
+ a.max = -4503599627370496
123
+ for i, value in ipairs(data) do
124
+ n = get_value(value)
125
+ if n > a.max then
126
+ a.max = n
127
+ end
128
+ end
129
+ else
130
+ redis.log(redis.LOG_ERROR, "Not implemented")
131
+ end
132
+ end
133
+ end
134
+
135
+ -- Update the buckets
136
+ for i, method in ipairs(config["aggregations"]) do
137
+ rra_key = key..'_'..rra["steps"]..'_'..method
138
+ -- We may be updating an old entry, which we want to delete first
139
+ redis.call("ZREMRANGEBYSCORE", rra_key, higher_ts, higher_ts)
140
+ -- Get the amount of entries in this bucket
141
+ count = redis.call("ZCARD", rra_key)
142
+
143
+ if count > higher["rows"] then
144
+ -- We have too many entries in this bucket - remove the oldest
145
+ oldest = redis.call("ZRANGE", rra_key, 0, 0)
146
+ redis.call("ZREM", rra_key, oldest[1])
147
+ end
148
+
149
+ -- Add the new entry to the bucked
150
+ if method == "average" then
151
+ redis.call("ZADD", rra_key, higher_ts, higher_ts..'_'..a.avg)
152
+ elseif method == "sum" then
153
+ redis.call("ZADD", rra_key, higher_ts, higher_ts..'_'..a.sum)
154
+ elseif method == "min" then
155
+ redis.call("ZADD", rra_key, higher_ts, higher_ts..'_'..a.min)
156
+ elseif method == "max" then
157
+ redis.call("ZADD", rra_key, higher_ts, higher_ts..'_'..a.max)
158
+ end
159
+
160
+ last_rra = rra
161
+ end
162
+
163
+ -- Set the higher precision bucket to the current rra
164
+ higher = last_rra
165
+ higher_key = key..'_'..higher["steps"]
166
+ end
167
+ end
168
+ end
169
+
170
+ return true
@@ -0,0 +1,61 @@
1
+ require 'json'
2
+ require 'redis'
3
+
4
+ class RReDis
5
+ VERSION = '0.1.0'
6
+ attr_reader :default_config
7
+
8
+ def initialize(options = {})
9
+ @r = Redis.new
10
+ @default_config = {:steps=>10, :rows=>8640,
11
+ :aggregations=>["average", "min", "max"],
12
+ :rra => [ {:steps=>60, :rows=>10080, :xff=>0.5},
13
+ {:steps=>900, :rows=>2976, :xff=>0.5},
14
+ {:steps=>3600, :rows=>8760, :xff=>0.5}]}
15
+ self.default_config = @default_config
16
+
17
+
18
+ @script_cache = {}
19
+ @sha_cache = {}
20
+ Dir.glob(File.join(File.dirname(__FILE__), '/lua/*.lua')).each do |file|
21
+ name = File.basename(file, File.extname(file))
22
+ @script_cache[name.to_sym] = File.open(file).read
23
+ @sha_cache[name.to_sym] = @r.script(:load, @script_cache[name.to_sym])
24
+ end
25
+
26
+ end
27
+
28
+ def default_config=(config)
29
+ self.config('default', config, false)
30
+ end
31
+
32
+ def config(metric, config, set_metric=true)
33
+ config[:rra] = config[:rra].sort {|a,b| a[:steps] <=> b[:steps] } if config[:rra]
34
+ @r.set("rrd_#{metric}_config", JSON.dump(config))
35
+ @r.sadd "rrd_metrics_set", metric if set_metric
36
+ end
37
+
38
+ def store(metric, timestamp, value=nil)
39
+ if value.nil?
40
+ value = timestamp
41
+ timestamp = Time.now
42
+ end
43
+ @r.evalsha(@sha_cache[:store], :keys => [metric], :argv => [value, timestamp.to_f])
44
+ end
45
+
46
+ def get(metric, start, stop, method = nil)
47
+ resp = @r.evalsha(@sha_cache[:get], :keys => [metric], :argv => [start.to_i, stop.to_i, method])
48
+ if resp
49
+ resp[1].collect! { |x| x.to_f}
50
+ resp
51
+ else
52
+ [[],[]]
53
+ end
54
+ end
55
+
56
+ def pipelined(&block)
57
+ @r.pipelined do
58
+ yield self
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,63 @@
1
+ require 'sinatra/base'
2
+ require 'slim'
3
+ require 'sprockets'
4
+
5
+ module RReDisServer
6
+ class SprocketsMiddleware
7
+ def initialize(app, options={})
8
+ @app = app
9
+ @root = options[:root]
10
+ path = options[:path] || 'assets'
11
+ @matcher = /^\/#{path}\/*/
12
+ @environment = ::Sprockets::Environment.new(@root)
13
+ @environment.append_path 'assets/javascripts'
14
+ @environment.append_path 'assets/javascripts/vendor'
15
+ @environment.append_path 'assets/stylesheets'
16
+ @environment.append_path 'assets/stylesheets/vendor'
17
+ @environment.append_path 'assets/images'
18
+ end
19
+
20
+ def call(env)
21
+ return [301, { 'Location' => "#{env['SCRIPT_NAME']}/" }, []] if env['SCRIPT_NAME'] == env['REQUEST_PATH']
22
+
23
+ return @app.call(env) unless @matcher =~ env["PATH_INFO"]
24
+ env['PATH_INFO'].sub!(@matcher,'')
25
+ @environment.call(env)
26
+ end
27
+ end
28
+
29
+ class Web < Sinatra::Base
30
+ dir = File.expand_path(File.dirname(__FILE__) + "/web")
31
+ set :views, "#{dir}/views"
32
+ set :root, "#{dir}/assets"
33
+ set :slim, :pretty => true
34
+ use SprocketsMiddleware, :root => dir
35
+
36
+
37
+ helpers do
38
+ def root_path
39
+ "#{env['SCRIPT_NAME']}/"
40
+ end
41
+ end
42
+
43
+ def initialize
44
+ super
45
+ @r = Redis.new
46
+ @rrd = RReDis.new
47
+ end
48
+
49
+ get "/" do
50
+ @metrics = @r.smembers("rrd_metrics_set")
51
+ slim :index
52
+ end
53
+
54
+ get "/get" do
55
+ data = {}
56
+ params['aggregations'].split(',').each do |method|
57
+ data[method] = @rrd.get(params['metric'], Time.now-params['timespan'].to_i, Time.now, method)
58
+ end
59
+ JSON.dump(data)
60
+ end
61
+ end
62
+
63
+ end
@@ -0,0 +1,3 @@
1
+ //= require vendor/jquery
2
+ //= require vendor/bootstrap
3
+ //= require_tree .
@@ -0,0 +1,97 @@
1
+ function RReDis() {
2
+ that = this;
3
+ this.charts = {};
4
+ this.config = [{name: "10 minutes", steps: 60*10}, {name: "1 hour", steps: 3600}, {name: "12 hours", steps: 3600*12}, {name: "1 day", steps: 3600*24},{name: "30 days", steps: 3600*24*31}];
5
+ $("#metrics").change(function(value) {
6
+ that.metric = $(this).val();
7
+ that.createGraphs(that.metric);
8
+ });
9
+ this.createGraphs = function(metric) {
10
+ var now = new Date().getTime()/1000;
11
+ $("#charts").html("")
12
+ var d = new Date();
13
+ for(var i = 0; i < this.config.length; i++) {
14
+ var config = this.config[i];
15
+ $("#charts").append('<div id="'+metric+config.steps+'" class="chart"></div>');
16
+ this.charts[metric+config.steps] = new Highcharts.Chart({
17
+ chart: {
18
+ renderTo: metric+config.steps,
19
+ defaultSeriesType: 'line',
20
+ events: {
21
+ load: this.requestData(metric, config)
22
+ },
23
+ zoomType: 'xy'
24
+
25
+ },
26
+ tooltip: {
27
+ xDateFormat: '%Y-%m-%d %H:%M:%S',
28
+ shared: true
29
+ },
30
+
31
+ title: {
32
+ text: config.name
33
+ },
34
+ xAxis: {
35
+ type: 'datetime',
36
+ tickPixelInterval : 50,
37
+ labels: {
38
+ rotation: -45,
39
+ align: 'right',
40
+ },
41
+ maxZoom: 20 * 1000,
42
+ max: d.getTime(),
43
+ min: d.getTime()-(config.steps*1000)
44
+ },
45
+ yAxis: {
46
+ minPadding: 0.2,
47
+ maxPadding: 0.2,
48
+ title: {
49
+ text: 'Value',
50
+ margin: 80
51
+ }
52
+ },
53
+ plotOptions: {
54
+ line: {
55
+ lineWidth: 1,
56
+ marker: {
57
+ enabled: false,
58
+ states: {
59
+ hover: {
60
+ enabled: true,
61
+ radius: 5
62
+ }
63
+ }
64
+ },
65
+ }
66
+ },
67
+ series: [{
68
+ name: metric,
69
+ data: [ ]
70
+ },
71
+ { name: metric + "min",
72
+ data: []},
73
+ { name: metric + "max",
74
+ data: []}]
75
+ });
76
+ }
77
+ };
78
+ this.requestData = function(metric, config) {
79
+ var that = this;
80
+ $.getJSON('get', {metric: metric, timespan: config.steps, aggregations: "average,min,max"},
81
+ function(data) {
82
+ var i = 0;
83
+
84
+ for(method in data) {
85
+ var items = [];
86
+ $.each(data[method][0], function(key, val) {
87
+ items.push([val*1000, data[method][1][key]]);
88
+ });
89
+
90
+ that.charts[metric+config.steps].series[i].setData (items, true, false);
91
+ i++;
92
+ }
93
+
94
+ }
95
+ );
96
+ };
97
+ }