redis_recipes 0.3.0

data/.rspec

@@ -0,0 +1 @@
+--colour

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    redis_recipes (0.3.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    rake (0.9.2.2)
+    redis (3.0.2)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.3)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+  redis (~> 3.0.0)
+  redis_recipes!
+  rspec

data/README.md

@@ -0,0 +1,26 @@
+# Redis Recipes
+
+Redis LUA recipes. Require Redis 2.6.0 or higher.
+
+## License
+
+Copyright (c) 2012 Black Square Media Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake'
+
+require 'rspec/mocks/version'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+desc 'Default: run specs.'
+task :default => :spec

data/lib/range_lookup/README.md

@@ -0,0 +1,62 @@
+# Redis Recipes: Range Lookup
+
+Data structure, optimised for fast lookups of (possibly overlapping) ranges
+containing a given value. Works with any numericly representable ranges, e.g.
+dates, times, IP addresses, etc. Inspired by: http://stackoverflow.com/questions/8622816/redis-or-mongo-for-determining-if-a-number-falls-within-ranges/8624231#8624231
+
+### Example / Internals:
+
+Consider the following use case: "Holiday System"
+
+* [A]lice is on vacation from the 4th until the 11th
+* [B]ob leaves an 7th and returns on the 21st
+* [C]olin is also away from 4th, but returns only on the 18th
+* [D]eborah's holidays are from 16th to the 23th
+
+Our numeric ranges would therefore be:
+
+    A  4-11
+    B  7-21
+    C  4-18
+    D 16-23
+
+To identify the people away for any specific date quickly, we create a
+flattened index:
+
+     4  [A C]
+     7  [A B C]
+    11  [B C]
+    16  [B C D]
+    18  [B D]
+    21  [D]
+    23  []
+
+For any given day, we can match the list (set) of members that are on vacation,
+by:
+
+1. Just reading the members for exact hits. For example: on the 18th [B C D]
+match.
+2. Creating an intersection between the preceeding and the following set. For
+example: on the 19th INTER([B C D], [B D]) -> [B D] match.
+
+### Usage:
+
+Add ranges:
+
+    redis-cli --eval <path/to/add.lua> holidays , A 4 11
+    redis-cli --eval <path/to/add.lua> holidays , B 7 21
+    redis-cli --eval <path/to/add.lua> holidays , C 4 18
+    redis-cli --eval <path/to/add.lua> holidays , D 16 23
+    redis-cli --eval <path/to/add.lua> holidays , X 5 15
+
+Remove a range:
+
+    redis-cli --eval <path/to/remove.lua> holidays , X 5 15
+
+Lookup range:
+
+    redis-cli --eval <path/to/lookup.lua> holidays , 18   # =>  1) "B"
+                                                          #     2) "C"
+                                                          #     3) "D"
+    redis-cli --eval <path/to/lookup.lua> holidays , 19   # =>  1) "B"
+                                                          #     2) "D"

data/lib/range_lookup/add.lua

@@ -0,0 +1,66 @@
+if #KEYS ~= 1 or #ARGV ~= 3 then
+  return redis.error_reply("wrong number of arguments")
+end
+
+local prefix = KEYS[1]
+local member = ARGV[1]
+local min    = tonumber(ARGV[2])
+local max    = tonumber(ARGV[3])
+
+if min == nil or max == nil or min > max then
+  return redis.error_reply("min/max are not numeric or out of range")
+end
+
+local index  = prefix .. ":~"
+local minscr = redis.call('zscore', index, min)
+local maxscr = redis.call('zscore', index, max)
+local minwrp = {}
+local maxwrp = {}
+local window = redis.call('zrangebyscore', index, "(" .. min, "(" .. max)
+
+-- Find existing members to be included in the new min
+if not minscr then
+  local before = redis.call('zrevrangebyscore', index, "(" .. min, "-inf", "limit", 0, 1)[1]
+  if before then
+    local after = redis.call('zrangebyscore', index, "(" .. min, "inf", "limit", 0, 1)[1]
+    if after then
+      minwrp  = redis.call('sinter', prefix .. ":" .. before, prefix .. ":" .. after)
+    end
+  end
+end
+
+-- Find existing members to be included in the new max
+if not maxscr then
+  local after = redis.call('zrangebyscore', index, "(" .. max, "inf", "limit", 0, 1)[1]
+  if after then
+    local before = redis.call('zrevrangebyscore', index, "(" .. max, "-inf", "limit", 0, 1)[1]
+    if before then
+      maxwrp  = redis.call('sinter', prefix .. ":" .. before, prefix .. ":" .. after)
+    end
+  end
+end
+
+-- Store members in min & max sets
+redis.call('sadd', prefix .. ":" .. min, member)
+redis.call('sadd', prefix .. ":" .. max, member)
+
+-- Store new min & max indices
+if not minscr then redis.call('zadd', index, min, min) end
+if not maxscr then redis.call('zadd', index, max, max) end
+
+-- Store member in all existing sets between min & max
+for _,key in pairs(window) do
+  redis.call('sadd', prefix .. ":" .. key, member)
+end
+
+-- Merge existing members into min
+if #minwrp > 0 then
+  redis.call('sadd', prefix .. ":" .. min, unpack(minwrp))
+end
+
+-- Merge existing members into max
+if #maxwrp > 0 then
+  redis.call('sadd', prefix .. ":" .. max, unpack(maxwrp))
+end
+
+return redis.status_reply("OK")

data/lib/range_lookup/lookup.lua

@@ -0,0 +1,27 @@
+if #KEYS ~= 1 or #ARGV ~= 1 then
+  return redis.error_reply("wrong number of arguments")
+end
+
+local prefix = KEYS[1]
+local value  = tonumber(ARGV[1])
+
+if value == nil then
+  return redis.error_reply("value is not numeric or out of range")
+end
+
+local members = {}
+local score   = redis.call('zscore', prefix .. ":~", value)
+
+if score then -- Do we have an exact match?
+  members = redis.call('smembers', prefix .. ":" .. score)
+else
+  local before  = redis.call('zrevrangebyscore', prefix .. ":~", value, "-inf", "limit", 0, 1)[1]
+  if before then
+    local after = redis.call('zrangebyscore', prefix .. ":~", value, "inf", "limit", 0, 1)[1]
+    if after then
+      members = redis.call('sinter', prefix .. ":" .. before, prefix .. ":" .. after)
+    end
+  end
+end
+
+return members

data/lib/range_lookup/remove.lua

@@ -0,0 +1,62 @@
+if #KEYS ~= 1 or #ARGV ~= 3 then
+  return redis.error_reply("wrong number of arguments")
+end
+
+local prefix = KEYS[1]
+local member = ARGV[1]
+local min    = tonumber(ARGV[2])
+local max    = tonumber(ARGV[3])
+
+if min == nil or max == nil or min > max then
+  return redis.error_reply("min/max are not numeric or out of range")
+end
+
+local index  = prefix .. ":~"
+local window = redis.call('zrangebyscore', index, min, max)
+local before = nil
+local after  = nil
+local minwrp = {}
+local maxwrp = {}
+
+-- Remove the member from all sets between min & max
+for _, val in pairs(window) do
+  redis.call('srem', prefix .. ":" .. val, member)
+end
+
+-- Identify members wrapped in min
+before = redis.call('zrevrangebyscore', index, "(" .. min, "-inf", "limit", 0, 1)[1]
+if before then
+  after = redis.call('zrangebyscore', index, "(" .. min, "inf", "limit", 0, 1)[1]
+  if after then
+    minwrp = redis.call('sinter', prefix .. ":" .. before, prefix .. ":" .. after)
+  end
+end
+
+-- Identify members wrapped in max
+after = redis.call('zrangebyscore', index, "(" .. max, "inf", "limit", 0, 1)[1]
+if after then
+  before = redis.call('zrevrangebyscore', index, "(" .. max, "-inf", "limit", 0, 1)[1]
+  if before then
+    maxwrp = redis.call('sinter', prefix .. ":" .. before, prefix .. ":" .. after)
+  end
+end
+
+-- Remove existing wrapped members from min
+if #minwrp > 0 then
+  redis.call('srem', prefix .. ":" .. min, unpack(minwrp))
+end
+
+-- Remove existing wrapped members from max
+if #maxwrp > 0 then
+  redis.call('srem', prefix .. ":" .. max, unpack(maxwrp))
+end
+
+-- Remove the min index, if no more items are left in the min set
+local minlen = redis.call('scard', prefix .. ":" .. min)
+if minlen == 0 then redis.call('zrem', index, min) end
+
+-- Remove the max index, if no more items are left in the max set
+local maxlen = redis.call('scard', prefix .. ":" .. max)
+if maxlen == 0 then redis.call('zrem', index, max) end
+
+return redis.status_reply("OK")

data/lib/range_lookup_x/README.md

@@ -0,0 +1,59 @@
+# Redis Recipes: Range Lookup X
+
+A data structure, very similar to the "normal" Range Lookup, except that it 
+stores ranges that exclude the end value.
+
+### Example / Internals:
+
+Consider the following use case: "Holiday System"
+
+* [A]lice is on vacation from the 4th until the 11th
+* [B]ob leaves an 7th and returns on the 21st
+* [C]olin is also away from 4th, but returns only on the 18th
+* [D]eborah's holidays are from 16th to the 23th
+
+Our numeric ranges would therefore be:
+
+    A  4...12
+    B  7...22
+    C  4...19
+    D 16...24
+
+To identify the people away for any specific date quickly, we create a
+flattened index:
+
+     4  [A C]
+     7  [A B C]
+    12  [B C]
+    16  [B C D]
+    19  [B D]
+    22  [D]
+    24  []
+
+For any given day, we can match the list (set) of members that are on vacation,
+by:
+
+1. Finding the index <= the given value. For example: a search for the 18th would return 16.
+2. Reading the members on the index. For example: for the 16th, [B C D] would be returned.
+
+### Usage:
+
+Add ranges:
+
+    redis-cli --eval <path/to/add.lua> holidays , A 4 12
+    redis-cli --eval <path/to/add.lua> holidays , B 7 22
+    redis-cli --eval <path/to/add.lua> holidays , C 4 19
+    redis-cli --eval <path/to/add.lua> holidays , D 16 24
+    redis-cli --eval <path/to/add.lua> holidays , X 5 16
+
+Remove a range:
+
+    redis-cli --eval <path/to/remove.lua> holidays , X 5 16
+
+Lookup range:
+
+    redis-cli --eval <path/to/lookup.lua> holidays , 18   # =>  1) "B"
+                                                          #     2) "C"
+                                                          #     3) "D"
+    redis-cli --eval <path/to/lookup.lua> holidays , 19   # =>  1) "B"
+                                                          #     2) "D"

data/lib/range_lookup_x/add.lua

@@ -0,0 +1,59 @@
+if #KEYS ~= 1 or #ARGV ~= 3 then
+  return redis.error_reply("wrong number of arguments")
+end
+
+local prefix = KEYS[1]
+local member = ARGV[1]
+local min    = tonumber(ARGV[2])
+local max    = tonumber(ARGV[3])
+
+if min == nil or max == nil or min >= max then
+  return redis.error_reply("min/max are not numeric or out of range")
+end
+
+local index  = prefix .. ":~"
+local minscr = redis.call('zscore', index, min)
+local maxscr = redis.call('zscore', index, max)
+local minwrp = {}
+local maxwrp = {}
+local window = redis.call('zrangebyscore', index, "(" .. min, "(" .. max)
+
+-- Find existing members to be included in the new min
+if not minscr then
+  local before = redis.call('zrevrangebyscore', index, "(" .. min, "-inf", "limit", 0, 1)[1]
+  if before then
+    minwrp = redis.call('smembers', prefix .. ":" .. before)
+  end
+end
+
+-- Find existing members to be included in the new max
+if not maxscr then
+  local before = redis.call('zrevrangebyscore', index, "(" .. max, "-inf", "limit", 0, 1)[1]
+  if before then
+    maxwrp = redis.call('smembers', prefix .. ":" .. before)
+  end
+end
+
+-- Store members in min set
+redis.call('sadd', prefix .. ":" .. min, member)
+
+-- Store new min & max indices
+if not minscr then redis.call('zadd', index, min, min) end
+if not maxscr then redis.call('zadd', index, max, max) end
+
+-- Store member in all existing sets between min & max
+for _,key in pairs(window) do
+  redis.call('sadd', prefix .. ":" .. key, member)
+end
+
+-- Merge existing members into min
+if #minwrp > 0 then
+  redis.call('sadd', prefix .. ":" .. min, unpack(minwrp))
+end
+
+-- Merge existing members into max
+if #maxwrp > 0 then
+  redis.call('sadd', prefix .. ":" .. max, unpack(maxwrp))
+end
+
+return redis.status_reply("OK")

data/lib/range_lookup_x/lookup.lua

@@ -0,0 +1,19 @@
+if #KEYS ~= 1 or #ARGV ~= 1 then
+  return redis.error_reply("wrong number of arguments")
+end
+
+local prefix = KEYS[1]
+local value  = tonumber(ARGV[1])
+
+if value == nil then
+  return redis.error_reply("value is not numeric or out of range")
+end
+
+local members = {}
+local pos     = redis.call('zrevrangebyscore', prefix .. ":~", value, "-inf", "limit", 0, 1)[1]
+
+if pos then
+  members = redis.call('smembers', prefix .. ":" .. pos)
+end
+
+return members

data/lib/range_lookup_x/remove.lua

@@ -0,0 +1,51 @@
+if #KEYS ~= 1 or #ARGV ~= 3 then
+  return redis.error_reply("wrong number of arguments")
+end
+
+local prefix = KEYS[1]
+local member = ARGV[1]
+local min    = tonumber(ARGV[2])
+local max    = tonumber(ARGV[3])
+
+if min == nil or max == nil or min >= max then
+  return redis.error_reply("min/max are not numeric or out of range")
+end
+
+local index   = prefix .. ":~"
+local window  = redis.call('zrangebyscore', index, min, "(" .. max)
+
+local minlen  = redis.call('scard', prefix .. ":" .. min)
+local maxlen  = redis.call('scard', prefix .. ":" .. max)
+
+-- Calculate cardinality of the set before min
+local befmin  = redis.call('zrevrangebyscore', index, "(" .. min, "-inf", "limit", 0, 1)[1]
+local bminlen = 0
+if befmin then
+  bminlen = redis.call('scard', prefix .. ":" .. befmin)
+end
+
+-- Calculate cardinality of the set before max
+local befmax  = redis.call('zrevrangebyscore', index, "(" .. max, "-inf", "limit", 0, 1)[1]
+local bmaxlen = 0
+if befmax then
+  bmaxlen = redis.call('scard', prefix .. ":" .. befmax)
+end
+
+-- Remove min if the cardinality between min and the set before min differs by 1
+if minlen - bminlen == 1 then
+  redis.call('del', prefix .. ":" .. min)
+  redis.call('zrem', index, min)
+end
+
+-- Remove max if the cardinality between max and the set before max differs by -1
+if bmaxlen - maxlen  == 1 then
+  redis.call('del', prefix .. ":" .. max)
+  redis.call('zrem', index, max)
+end
+
+-- Remove the member from all sets between min & max
+for _, val in pairs(window) do
+  redis.call('srem', prefix .. ":" .. val, member)
+end
+
+return redis.status_reply("OK")