queris 0.8.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3308c207243318c193dcc8557c6b5120bff0fb85
+  data.tar.gz: 0b81f79d85f4adc0fc801b671d92b342b6ebdf8f
+SHA512:
+  metadata.gz: a59b24e659d0652dd1403682720fc75b8de4e91f57ba0013fe1da3a254de2aaa562cc9e627515e1e81272793dae2a85ab27eb101c349df624cf98c4ba3859530
+  data.tar.gz: 0ef348c9862862fd5a0159b57e0efa64c9479427050a67911ec8d83c87bf4c0ba1a3d4592fc2904b7e368f8c320a537bd4ec704cec23eb8240577a6ec027508e

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in queris.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    queris (0.8.1)
+      diffy
+      hiredis
+      redis (~> 3.2.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    coderay (1.1.0)
+    diffy (3.0.7)
+    hiredis (0.6.0)
+    method_source (0.8.2)
+    pry (0.10.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-debundle (0.8)
+      pry
+    redis (3.2.1)
+    slop (3.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry
+  pry-debundle
+  queris!
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -0,0 +1,53 @@
+Nutshell
+========
+A Ruby object indexing and querying tool using Redis to store data and perform set operations.
+
+What's all this about?
+====
+Redis is a flexible data structures server. It's pretty fast, but if you want to index something, you have to do it manually.
+Queris can be bolted onto some Ruby objects (hashes, ActiveRecord models, Ohm objects, etc.) to maintain indices and perform queries. Attributes can be indexed into regular or sorted sets, allowing set selection on arbitrary data, and range selection on numeric data (if desired). More complex indexing, like n-gram text search, can be implemented client-side (and will be included in some future Queris version.)
+Queries can perform set operations on all indices and other queries, and can be sorted. Each query can be cached for a custom duration. Queries can also be live (reflect realtime changes to complex set operations) in O(log(n)) server time.
+Thus Queris offers customizeable indexing, and nestable, cacheable, optionally realtime, set queries (with a few more bells and whistles).
+
+Learning By Example
+===================
+First connect to redis. Let's assume a Redis server on localhost at standard port 6379:
+```ruby
+  Queris.add_redis Redis.new
+```
+Let's say you have a User ActiveRecord model with some obvious attributes - id, name, email, age. Let's also assume a UserTags model, linked to User by its userId attribute.
+```ruby
+class User < ActiveRecord::Base
+  include Queris
+  
+  #declare which attributes to index, and how
+  index_attributes :name, :email   #simple set index
+  index_range_attribute :age   #sorted set index
+  index_attribute_from model: UserTags, attribute: :tag, key: :userId  #index from a different model
+end
+```
+To build indices:
+```ruby
+  User.build_redis_indices
+```
+This will load all current User objects into memory and index said objects. You currently need enough space to keep the entire dataset in memory for building indices. (Future versions will be able to import data incrementally.)
+
+You can now query Users:
+```ruby
+young_bob_and_steve = User.query(:ttl=>2.days).union(:name, ["Steve", "Bob"]).intersect(:age, 0..30).diff(:email, "steve@example.org") #query expires in 2 days
+#you can have subqueries, too:
+bob_and_steve_and_bill = User.query.union(young_bob_and_steve).union(:name, "Bill")
+#now get the results
+bob_and_steve_and_bill.results #all results
+bob_and_steve_and_bill.count #result count
+bob_and_steve_and_bill.results(3..5) # results # 3 through 5
+```
+Queries can be expressed in set notation:
+```ruby
+bob_and_steve_and_bill.to_s # same as .explain
+#  ((name<["Steve", "Bob"]> ∩ age<0..30> ∖ email<steve@example.org>) ∪ name<Bill>)
+```
+
+Note that query operations are applied sequentially (with no complex operator precedence rules), but can be arranged or grouped by the use of subqueries.
+
+...more to follow...

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/data/redis_scripts/add_low_ttl.lua

@@ -0,0 +1,10 @@
+local min_ttl=tonumber(ARGV[1])
+local keys=KEYS
+local state=table.remove(keys) --last key is state key
+for i, k in ipairs(keys) do
+  local ttl = tonumber(redis.call('ttl', k))
+  if ttl>0 and ttl < min_ttl then
+    redis.call('expire', k, ttl + min_ttl)
+    table.sadd(state, k)
+  end
+end

data/data/redis_scripts/copy_key_if_absent.lua

@@ -0,0 +1,13 @@
+local dst, src = KEYS[1], KEYS[2]
+local t = redis.call('type', src).ok
+if t == 'zset' then
+  redis.call('zunionstore', dst, 1, src)
+elseif t == 'set' then
+  redis.call('sunionstore', dst, 1, src)
+elseif t == 'string' then
+  redis.call('set', dst, redis.call('get', src))
+elseif t == 'none' then
+  --noop
+else
+  redis.log(redis.LOG_WARNING, "copy_key_if_absent: unsupported key type " .. t .. ".")
+end

data/data/redis_scripts/copy_ttl.lua

@@ -0,0 +1,13 @@
+if redis.call('exists', KEYS[2]) == 0 then
+  return nil
+end
+local ttl, delete_if_absent = redis.call("ttl", KEYS[1]), (ARGV[1] or false)
+if ttl ~= -1 then
+  redis.call("expire", KEYS[2], ttl)
+  return ttl
+elseif delete_if_absent and redis.call('exists', KEYS[1]) == 0 then
+  if redis.call("del", KEYS[2]) == 1 then
+    return "DELETED"
+  end
+end
+return 'OK'

data/data/redis_scripts/create_page_if_absent.lua

@@ -0,0 +1,24 @@
+local dst, source = KEYS[1], KEYS[2]
+local min, max = ARGV[1], ARGV[2]
+local dst_type = redis.call('type', dst).ok
+redis.log(redis.LOG_WARNING, "want a page at " .. dst .. " from " .. source)
+if dst_type == 'string' then
+  redis.call('del', dst)
+elseif dst_type == 'zset' then
+  redis.log(redis.LOG_WARNING, 'page ' .. dst .. ' already exists with size ' .. redis.call('zcard', dst))
+  return redis.call('zcard', dst)
+end
+
+local res = redis.call("zrange", source, min, max, "withscores")
+local id, score
+if next(res) ~= nil then
+  redis.call('del', dst)
+end
+for i,v in ipairs(res) do
+  if i%2==1 then id=v; else
+    score=v
+    redis.call('zadd', dst, score, id)
+  end
+end
+redis.log(redis.LOG_WARNING, 'pagesize of ' .. dst .. ': ' .. redis.call('zcard', dst))
+return #res/2 

data/data/redis_scripts/debuq.lua

@@ -0,0 +1,20 @@
+local cmd=ARGV[1]
+local keys = KEYS
+
+local info = function(k)
+  local t = redis.call('type', k).ok
+  if t == 'zset' then
+    return "zset size=" .. redis.call('zcard', k)
+  elseif t== 'set' then
+    return "set size=" .. redis.call('scard', k)
+  elseif t== 'string' then
+    return "string = " .. redis.call('get', k)
+  else
+    return t
+  end
+end
+
+redis.log(redis.LOG_WARNING, "info for " ..cmd.. ": " .. #KEYS .. " keys")
+for k,v in pairs(keys) do
+  redis.log(redis.LOG_WARNING, ("  " .. v .. ": " .. info(v)))
+end

data/data/redis_scripts/delete_if_string.lua

@@ -0,0 +1,8 @@
+local count = 0
+for i,key in ipairs(KEYS) do
+  if redis.call('type', key).ok == "string" then
+    redis.call('del', key)
+    count = count + 1
+  end
+end
+return count

data/data/redis_scripts/delete_matching_keys.lua

@@ -0,0 +1,7 @@
+local keymatch = ARGV[1]
+local keys = redis.call('keys', keymatch)
+local deleted = 0
+for i,key in ipairs(keys) do
+  deleted = deleted + redis.call('del', key)
+end
+return deleted

data/data/redis_scripts/expire_temp_query_keys.lua

@@ -0,0 +1,7 @@
+local ttl = ARGV[1]
+for i,k in ipairs(KEYS) do
+  if redis.call('exists', k) ~= 1 then
+    redis.call('setnx', k, 1337)
+    redis.call('expire', k, ttl)
+  end
+end

data/data/redis_scripts/make_rangehack_if_needed.lua

@@ -0,0 +1,30 @@
+local dst, src, rangehack_set_key, not_needed_key = KEYS[1], KEYS[2], KEYS[3], KEYS[4]
+local min, max, exclude_max = tonumber(ARGV[1]), tonumber(ARGV[2]), ARGV
+redis.log(redis.LOG_WARNING, "want rangehack ("..min..","..max..") for " .. src)
+if redis.call('exists', not_needed_key) == 1 then
+  --nevermind, we don't need to run this
+  redis.log(redis.LOG_WARNING, "nevermind, the query doesn't need it..")
+  return
+end
+local opf = "(%s" --)
+local t = redis.call('type', dst).ok
+if t == 'zset' then
+  --already exists, nothing to do
+  redis.log(redis.LOG_WARNING, "it's already here.")
+  return
+else
+  redis.call('zunionstore', dst, 1, src) --slow as a fat, barbed turd
+  --remove inverse range
+  redis.call('zremrangebyscore', dst, '-inf', opf:format(min))
+  redis.call('zremrangebyscore', dst, (exclude_max and max or opf:format(max)), 'inf')
+  
+  --add to rangehack 'index'
+  redis.call('sadd', rangehack_set_key, dst)
+  
+  redis.log(redis.LOG_WARNING, "made one at " .. dst .. " with size " .. redis.call('zcard', dst))
+  if redis.call('zcard', dst) == 0 then
+    --dummy one-element zset
+    redis.call('zadd', dst, 0, "=-{*_*}-= <- the nullset ghost")
+  end
+  redis.log(redis.LOG_WARNING, "made one at " .. dst .. " with size " .. redis.call('zcard', dst))
+end

data/data/redis_scripts/master_expire.lua

@@ -0,0 +1,15 @@
+local ttl, renew, dummy = ARGV[1], ARGV[2], ARGV[3]
+redis.log(redis.LOG_WARNING, "master_expire keys [" .. table.concat(KEYS, ', ') .. "] with ttl " .. ttl .. " renew: " .. (renew and "yes" or "no") .. " dummy: " .. (dummy and "yes" or "no"))
+for i,k in ipairs(KEYS) do
+  local exists = redis.call('exists', k) == 1
+  if not exists and dummy then
+    exists = 1
+    redis.call('setnx', k, 'dummy')
+  end
+  if exists then
+    local current_ttl = redis.call('ttl', k)
+    if current_ttl < 0 or renew  then
+      redis.call('expire', k, ttl)
+    end
+  end
+end

data/data/redis_scripts/match_key_type.lua

@@ -0,0 +1,9 @@
+local key = KEYS[1]
+local acceptable_types = ARGV
+local keytype = redis.call('type', key).ok
+for i, t in ipairs(acceptable_types) do
+  if t == keytype then
+    return true
+  end
+end
+return false

data/data/redis_scripts/move_key.lua

@@ -0,0 +1,11 @@
+local oldkey, newkey = KEYS[1], KEYS[2]
+if redis.call('exists', oldkey) == 1 then
+  redis.call('rename', oldkey, newkey)
+  return 1
+else
+  --oldkey is empty, so we move that emptiness to newkey
+  if redis.call('exists', newkey) == 1 then
+    redis.call('del', newkey) --clear newkey
+  end
+  return 0
+end

data/data/redis_scripts/multisize.lua

@@ -0,0 +1,19 @@
+local key = KEYS[1]
+local keytype = redis.call('type', key).ok
+
+if keytype=='set' then
+  return redis.call('scard', key)
+elseif keytype=='zset' then
+  return redis.call('zcard', key)
+elseif keytype=='list' then
+  return redis.call('llen', key)
+elseif keytype=='hash' then
+  return redis.call('hlen', key)
+elseif keytype=='none' then
+  return 0
+elseif keytype=='string' then
+  return redis.call('strlen', key)
+else
+  redis.log(redis.LOG_WARNING, "I wonder what the size of key " .. key .. "(" .. keytype .. ")".. "is?")
+  return "WTF for " .. keytype
+end

data/data/redis_scripts/paged_query_ready.lua

@@ -0,0 +1,35 @@
+local results_key, exists_key, sort_key, page_key, runstate_key = KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5]
+local pagesize, min, max = tonumber(ARGV[1]), tonumber(ARGV[2]), tonumber(ARGV[3])
+
+if redis.call('exists', exists_key) ~= 1 then
+  return nil
+end
+
+local last_loaded_page = redis.call('get', page_key)
+
+if not last_loaded_page then
+  return nil
+end
+
+local t = redis.call('type', results_key).ok
+local current_count
+if t == 'zset' then
+  current_count = redis.call('zcard', results_key)
+elseif t=='set' then
+  current_count = redis.call('scard', results_key)
+else
+  current_count = 0
+end
+
+if current_count > max then
+  if runstate_key then
+    redis.call('set', runstate_key, 1)
+  end
+  return true
+elseif (last_loaded_page + 1) * pagesize >= redis.call('zcard', sort_key) then
+  --we've reached the end
+  if runstate_key then
+    redis.call('set', runstate_key, 1)
+  end
+  return true
+end

data/data/redis_scripts/periodic_zremrangebyscore.lua

@@ -0,0 +1,9 @@
+local ttl = tonumber(ARGV[1])
+local key = KEYS[1]
+local timer_key = key .. ":timer"
+if redis.call('exists', timer_key) == 1 then
+  return 0
+else
+  redis.call('setex', timer_key, ttl, "don't do it as long as i exist")
+end
+return redis.call('zremrangebyscore', key, ARGV[2], ARGV[3])

data/data/redis_scripts/persist_reusable_temp_query_keys.lua

@@ -0,0 +1,14 @@
+local dst=table.remove(KEYS, 1)
+if redis.call('exists', dst) == 1 then
+  redis.log(redis.LOG_WARNING, "persist_reusable_temp_query_keys storage key " .. dst .. " already exist. WTF?")
+  redis.call('del', dst)
+end
+for i,k in ipairs(KEYS) do
+  local t= redis.call('type', k).ok
+  if t ~= 'none' then
+    ttl = redis.call('ttl', k)
+    redis.call('persist', k)
+    redis.zadd(dst, ttl, k)
+    redis.log(redis.LOG_WARNING, "persisted key " .. k .. " ttl=" .. ttl)
+  end
+end

data/data/redis_scripts/query_ensure_existence.lua

@@ -0,0 +1,23 @@
+local existence_key, results_key = KEYS[1], KEYS[2]
+local ttl, min_ttl, master_only = ARGV[1], tonumber(ARGV[2]), ARGV[3]=='true'
+if redis.call('exists', existence_key) == 1 then
+  for i, k in ipairs(KEYS) do
+    if min_ttl then
+      local key_ttl = redis.call('ttl', k)
+      if key_ttl < min_ttl then
+        redis.call('expire', k, min_ttl)
+      end
+    else
+      redis.call('expire', k, ttl)
+    end
+  end
+  if not master_only then
+    return true
+  else
+    return redis.call('type', results_key).ok ~= 'string'
+  end
+else
+  redis.call('setnx', results_key, 1)
+  redis.call('expire', results_key, ttl)
+  return false
+end

data/data/redis_scripts/query_intersect_optimization.lua

@@ -0,0 +1,31 @@
+local dstkey, key, smallkey = KEYS[1], KEYS[2], KEYS[3]
+local ttl = ARGV[1] or 120
+local t=redis.call('type', dstkey).ok
+
+local multisize = function(key)
+  local keytype = redis.call('type', key).ok
+  if keytype=='set' then
+    return "set card=" .. redis.call('scard', key)
+  elseif keytype=='zset' then
+    return "zset card=" .. redis.call('zcard', key)
+  elseif keytype=='list' then
+    return "list card=" .. redis.call('llen', key)
+  elseif keytype=='hash' then
+    return "hash card=" .. redis.call('hlen', key)
+  elseif keytype=='none' then
+    return "none"
+  elseif keytype=='string' then
+    return "string length=" .. redis.call('strlen', key)
+  else
+    return "WTF for " .. keytype
+  end
+end
+if t ~= 'zset' then
+  redis.call('zinterstore', dstkey, 2, key, smallkey, 'weights', 1, 0)
+  redis.call('expire', dstkey, ttl)
+  for i,v in pairs({dstkey, key, smallkey}) do
+    redis.log(redis.LOG_WARNING, " inter: " .. v .. ": " .. multisize(v))
+  end
+else
+  redis.log(redis.LOG_WARNING, "intersectoptimization " .. dstkey .. " already exists")
+end

data/data/redis_scripts/remove_from_keyspace.lua

@@ -0,0 +1,27 @@
+local element, keysearch, ttl = ARGV[1], ARGV[2], tonumber(ARGV[3])
+local keys_key = "Queris:removing_from_keyspace:" .. keysearch
+local keys
+
+--relevant keys
+if redis.call('zcard', keys_key) > 0 then
+  keys = redis.call('zrange', keys_key, 0, -1)
+else
+  keys = redis.call('keys', keysearch)
+  if #keys > 0 then
+    for i,v in ipairs(keys) do
+      redis.call('zadd', keys_key, 0, v)
+    end
+    redis.call('expire', keys_key, ttl)
+  end
+end
+
+local removed = 0
+for i,key in ipairs(keys) do
+  local keytype= redis.call('type', key).ok
+  if     keytype == 'zset' then
+    removed = removed + redis.call('zrem', key, element)
+  elseif keytype == 'set'  then
+    removed = removed + redis.call('srem', key, element)
+  end
+end
+return removed