rredis 0.1.0

data/.autotest

@@ -0,0 +1,23 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+require 'autotest/fsevent'
+# Autotest.add_hook :initialize do |at|
+#   at.extra_files << "../some/external/dependency.rb"
+#
+#   at.libs << ":../some/external"
+#
+#   at.add_exception 'vendor'
+#
+#   at.add_mapping(/dependency.rb/) do |f, _|
+#     at.files_matching(/test_.*rb$/)
+#   end
+#
+#   %w(TestA TestB).each do |klass|
+#     at.extra_class_map[klass] = "test/test_misc.rb"
+#   end
+# end
+
+# Autotest.add_hook :run_command do |at|
+#   system "rake build"
+# end

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+=== 0.1.0 / 2012-06-22
+
+  First release

data/Gemfile

@@ -0,0 +1,18 @@
+source :rubygems
+gem 'redis', '>=3.0.1'
+gem 'slim'
+gem 'sprockets'
+gem 'sass'
+gem 'vegas'
+
+group :test, :development do 
+  gem 'rspec'
+  gem 'simplecov', :require => false
+  gem 'hoe'
+  gem 'hoe-git'
+  gem 'hoe-gemspec'
+  gem 'hoe-bundler'
+  gem 'ZenTest'
+  gem 'autotest-fsevent'
+end
+

data/Gemfile.lock

@@ -0,0 +1,68 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    ZenTest (4.8.1)
+    autotest-fsevent (0.2.8)
+      sys-uname
+    diff-lcs (1.1.3)
+    ffi (1.0.11)
+    ffi (1.0.11-java)
+    hike (1.2.1)
+    hoe (3.0.6)
+      rake (~> 0.8)
+    hoe-bundler (1.1.0)
+      hoe (>= 2.2.0)
+    hoe-gemspec (1.0.0)
+      hoe (>= 2.2.0)
+    hoe-git (1.5.0)
+      hoe (>= 2.2.0)
+    multi_json (1.3.6)
+    rack (1.4.1)
+    rake (0.9.2.2)
+    redis (3.0.1)
+    rspec (2.10.0)
+      rspec-core (~> 2.10.0)
+      rspec-expectations (~> 2.10.0)
+      rspec-mocks (~> 2.10.0)
+    rspec-core (2.10.1)
+    rspec-expectations (2.10.0)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.10.1)
+    sass (3.1.19)
+    simplecov (0.6.4)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.5.3)
+    simplecov-html (0.5.3)
+    slim (1.2.2)
+      temple (~> 0.4.0)
+      tilt (~> 1.3.3)
+    sprockets (2.4.3)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    sys-uname (0.9.0)
+      ffi (>= 1.0.0)
+    temple (0.4.0)
+    tilt (1.3.3)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  ZenTest
+  autotest-fsevent
+  hoe
+  hoe-bundler
+  hoe-gemspec
+  hoe-git
+  redis (>= 3.0.1)
+  rspec
+  sass
+  simplecov
+  slim
+  sprockets
+  vegas

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2012-03-28
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,40 @@
+.autotest
+.rspec
+CHANGELOG.md
+Gemfile
+Gemfile.lock
+History.txt
+Manifest.txt
+README.md
+Rakefile
+benchmark/default.rb
+benchmark/pipeline.rb
+bin/rredis_server
+config.ru
+lib/lua/get.lua
+lib/lua/store.lua
+lib/rredis.rb
+lib/rredis/server.rb
+lib/rredis/web/assets/javascripts/application.js
+lib/rredis/web/assets/javascripts/main.js
+lib/rredis/web/assets/javascripts/vendor/bootstrap.js
+lib/rredis/web/assets/javascripts/vendor/bootstrap.min.js
+lib/rredis/web/assets/javascripts/vendor/highcharts/highcharts.js
+lib/rredis/web/assets/javascripts/vendor/highcharts/modules/canvas-tools.js
+lib/rredis/web/assets/javascripts/vendor/highcharts/modules/exporting.js
+lib/rredis/web/assets/javascripts/vendor/highcharts/themes/dark-blue.js
+lib/rredis/web/assets/javascripts/vendor/highcharts/themes/dark-green.js
+lib/rredis/web/assets/javascripts/vendor/highcharts/themes/gray.js
+lib/rredis/web/assets/javascripts/vendor/highcharts/themes/grid.js
+lib/rredis/web/assets/javascripts/vendor/highcharts/themes/skies.js
+lib/rredis/web/assets/javascripts/vendor/jquery.js
+lib/rredis/web/assets/stylesheets/application.css
+lib/rredis/web/assets/stylesheets/vendor/bootstrap-responsive.css
+lib/rredis/web/assets/stylesheets/vendor/bootstrap.css
+lib/rredis/web/images/glyphicons-halflings-white.png
+lib/rredis/web/images/glyphicons-halflings.png
+lib/rredis/web/views/index.slim
+lib/rredis/web/views/layout.slim
+spec/rredis/get_spec.rb
+spec/rredis/store_spec.rb
+spec/spec_helper.rb

data/README.md

@@ -0,0 +1,143 @@
+## RReDis
+
+https://github.com/dsander/RReDis
+
+RReDis is a round robin database backed by redis. It uses the brand new lua scripting feature of redis 2.6.
+
+
+## Getting started
+
+Install the gem
+
+    gem install rredis
+
+
+Store some metrics
+    
+    require 'rredis'
+    rrd = RReDis.new
+
+    # We start two hours in the past
+    start = (Time.now-(3600*2)).to_i
+
+    # We pretend to update the data every 10 seconds for two hours
+    (2*3600/10).times do |step|
+      rrd.store "example", start+step*10, rand(100)
+    end
+
+Fetch data from RReDis
+
+    require 'rredis'
+    rrd = RReDis.new
+
+    # Get the data from 5 minutes ago until now
+    puts rrd.get('example', Time.now-300, Time.now).inspect
+    # Get the data from one hour ago until 55 minutes ago
+    puts rrd.get('example', Time.now-3600, Time.now-3300, 'min').inspect
+    # Get the data from two hours ago until 90 minutes ago
+    puts rrd.get('example', Time.now-7200, Time.now-5400, 'max').inspect
+The get function takes three or four arguments:
+
+*  The metric to query
+*  Starting time stamp for the timespan to return
+*  Ending time stamp for the timespan to return
+*  Optional aggregation method 
+
+
+The array returned contains two arrays, the first one with unix timestamps of the measurements, the second one the (aggregated) values.
+
+## Configuration
+
+Per default RReDis is configured to store one measurement every 10 seconds for one day (called native resolution from here on), and then aggregate the measurements for the following timespans:
+
+*  1 week at 1 minute resolution
+*  1 month at 15 minute resolution
+*  1 year at 1 hour resolution
+
+RReDis also stores the average, minima and maxima of aggregated measurements.
+
+# Configuration format
+
+RReDis default configuration:
+
+    {:steps=>10, :rows=>17280, 
+     :aggregations=>["average", "min", "max"], 
+     :rra => [ {:steps=>60, :rows=>10080, :xff=>0.5},
+               {:steps=>900, :rows=>2976, :xff=>0.5},
+               {:steps=>3600, :rows=>8760, :xff=>0.5}]}
+
+`:steps` interval in seconds in which to store measurements
+
+`:rows`  amount of measurements to store, the timespan in seconds of the native resolution equals to `:steps`*`:rows`
+
+`:aggregations` array of aggregations to use for this measurement, currently available: `average`, `min`, `max`, `sum`
+
+`:rra`   array of archives to store historical data
+
+
+For every `:rra`:
+
+`:steps` interval in seconds in which to aggregate the measurements of the next higher resolution
+
+`:rows`  amount of aggregations to store, the timespan in seconds of the archive equals to `:steps`*`:rows`
+
+`:xff`   the xfiles factor - determines if aggregated measurements are stored. 1.0 would require 10 of 10 measurements from the next higher resolution (either form a rra or the native resolution), 0.1 would require 1 of 10 measurements
+
+# Modify the configuration
+You can either explicitly configure a metric via the `config` method:
+
+    rrd = RReDis.new  
+    rrd.config("metic", {:steps => 10, :rows => 3})
+
+
+Or change the default configuration which will be applied to every measurement without an explicit configuration:
+
+    rrd = RReDis.new
+    rrd.default_config = {:steps => 10, :rows => 3}
+
+
+## Performance
+
+Even though redis supports a lot of data types, it was clearly not made to act as a round robin database, but still with lua scripting the performance is really nice. 
+
+With the default configuration RReDis is able to handle around 2.5k updates per second (ubuntu vm with two cores of an i5 750 in virtual box running on windows), which is still amazingly fast considering the complexity of the store.lua script.
+
+Check out the default.rb script in the benchmark directory. Here are the results of my machine:
+
+    0.000395ms per op, 2529.6144076826317 op/s
+    415.557405432s for for 1051200 inserts
+    8639 stored measurements/aggregations
+    156 bytes used per stored measurement
+    6882960 measurements storable per gb of ram
+    794 metrics storable per gb of ram
+    5273285 redis commands performed
+    5 redis commands performed per stored measurement
+
+Using pipelining, we are able to determine the limits of redis itself. On my box redis can handle about 10k updates per second. A "advanced" key-value store able to run a lua script 10 thousand times a second doing about 50 thousand internal operations per second is just plain amazing. 
+Hats off! @antirez
+
+
+## LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2012 Dominik Sander
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,34 @@
+# -*- ruby -*-
+
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require "rspec/core/rake_task"
+
+# Hoe.plugin :compiler
+# Hoe.plugin :gem_prelude_sucks
+# Hoe.plugin :inline
+# Hoe.plugin :racc
+# Hoe.plugin :rubyforge
+Hoe.plugin :git
+Hoe.plugin :gemspec
+Hoe.plugin :bundler
+Hoe.plugin :gemcutter
+Hoe.plugins.delete :rubyforge
+
+Hoe.spec 'rredis' do
+  developer('Dominik Sander', 'git@dsander.de')
+  self.description ="RReDis - a round robin database backed by redis" 
+  self.summary = "RReDis - dead simple monitoring and graphing round robin database using redis"
+
+  self.readme_file = 'README.md'
+  self.history_file = 'CHANGELOG.md'
+  self.extra_deps << ["redis"]
+end
+
+task :prerelease => [:clobber, :check_manifest, :test]
+
+
+task :default => :spec
+task :test => :spec
+

data/benchmark/default.rb

@@ -0,0 +1,62 @@
+require 'rredis'
+require 'benchmark'
+
+r = Redis.new
+r.flushall
+r.config 'resetstat'
+
+rrd      = RReDis.new
+memory   = r.info["used_memory"].to_i
+commands = r.info["total_commands_processed"].to_i
+
+# Define who much data we want to store
+days  = 365
+steps = 30
+
+# Calculate the amount of measurements we need to store
+day   = 3600*24
+start = Time.now-(day*days)
+n     = day*days/steps
+
+# We regenerate the data to just measure the inserts
+data = []
+n.times do |i|
+  data << [(start+(i*steps)).to_f, rand(100_0)]
+end
+
+# Do the actual work
+res = Benchmark.realtime do
+  #rrd.pipelined do
+  n.times do |i|
+    d = data.pop
+    rrd.store "bench", d[0], d[1]
+  end
+  #end
+end
+
+puts "#{res/n}ms per op, #{1/(res/n)} op/s"
+
+# Calculate the amount of measurements we actually stored
+stored_meassuremnts = 0
+r.keys("rrd_bench*").each do |k|
+  next if k.include? 'config'
+  if !k.include? 'min' and !k.include? 'max'
+    stored_meassuremnts += r.zcard(k) 
+  end
+end
+
+used_memory               = r.info["used_memory"].to_i - memory
+commands_processed        = r.info["total_commands_processed"].to_i - commands
+
+bytes_per_measurement     = used_memory/stored_meassuremnts
+stored_measuremnts_per_gb = (1*1024*1024*1024).to_f/bytes_per_measurement
+stored_metrics_per_gb     = (1*1024*1024*1024).to_f/used_memory
+commands_per_measurement  =  commands_processed/n
+
+puts "#{res}s for for #{n} inserts"
+puts "#{stored_meassuremnts} stored measurements/aggregations"
+puts "#{bytes_per_measurement} bytes used per stored measurement"
+puts "#{stored_measuremnts_per_gb.to_i} measurements storable per gb of ram"
+puts "#{stored_metrics_per_gb.to_i} metrics storable per gb of ram"
+puts "#{commands_processed} redis commands performed"
+puts "#{commands_per_measurement} redis commands performed per stored measurement"

data/benchmark/pipeline.rb

@@ -0,0 +1,57 @@
+require 'rredis'
+require 'benchmark'
+
+r = Redis.new
+r.flushall
+r.config 'resetstat'
+
+rrd      = RReDis.new
+memory   = r.info["used_memory"].to_i
+commands = r.info["total_commands_processed"].to_i
+
+# Define who much data we want to store
+days  = 32
+steps = 30
+
+# Calculate the amount of measurements we need to store
+day   = 3600*24
+start = Time.now-(day*days)
+n     = day*days/steps
+
+# We regenerate the data to just measure the inserts
+data = []
+n.times do |i|
+  data << [(start+(i*steps)).to_f, rand(100_0)]
+end
+
+# Do the actual work
+res = Benchmark.realtime do
+  rrd.pipelined do
+    n.times do |i|
+      d = data.pop
+      rrd.store "bench", d[0], d[1]
+    end
+  end
+end
+
+puts "#{res/n}ms per op, #{1/(res/n)} op/s"
+
+# Calculate the amount of measurements we actually stored
+stored_meassuremnts = 0
+r.keys("rrd_bench*").each do |k|
+  next if k.include? 'config'
+  if !k.include? 'min' and !k.include? 'max'
+    stored_meassuremnts += r.zcard(k) 
+  end
+end
+
+used_memory               = r.info["used_memory"].to_i - memory
+commands_processed        = r.info["total_commands_processed"].to_i - commands
+
+bytes_per_measurement     = used_memory/stored_meassuremnts
+stored_measuremnts_per_gb = (1*1024*1024*1024).to_f/bytes_per_measurement
+stored_metrics_per_gb     = (1*1024*1024*1024).to_f/used_memory
+commands_per_measurement  =  commands_processed/n
+
+puts "#{res}s for for #{n} inserts"
+puts "#{commands_processed/res} redis commands per second"