redcord 0.0.2.alpha → 0.1.2

data/lib/redcord/server_scripts/delete_hash.erb.lua

@@ -12,8 +12,8 @@ The number of keys deleted from Redis
 -- The arguments can be accessed by Lua using the KEYS global variable in the
 -- form of a one-based array (so KEYS[1], KEYS[2], ...).
 --
---   KEYS[1] = Model.name
---   KEYS[2] = id
+--   KEYS = id, hash_tag
+--   ARGV = Model.name index_attr_size range_index_attr_size [index_attr_key ...] [range_index_attr_key ...] [custom_index_name ...]
 <%= include_lua 'shared/index_helper_methods' %>
 
 -- Validate input to script before making Redis db calls
@@ -21,28 +21,37 @@ if #KEYS ~= 2 then
   error('Expected keys of be of size 2')
 end
 
-local model = KEYS[1]
-local id = KEYS[2]
+local model = ARGV[1]
+local id, hash_tag = unpack(KEYS)
 
 -- key = "#{model}:id:{id}"
 local key = model .. ':id:' .. id
 
+local index_attr_pos = 4
+local range_attr_pos = index_attr_pos + ARGV[2]
+local custom_index_pos = range_attr_pos + ARGV[3]
+
 -- Clean up id sets for both index and range index attributes
-local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
+local index_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
 if #index_attr_keys > 0 then
 -- Retrieve old index attr values so we can delete them in the attribute id sets
   local attr_vals = redis.call('hmget', key, unpack(index_attr_keys))
   for i=1, #index_attr_keys do
-    delete_id_from_index_attr(model, index_attr_keys[i], attr_vals[i], id)
+    delete_id_from_index_attr(hash_tag, model, index_attr_keys[i], attr_vals[i], id)
   end
 end
-local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
+local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_index_pos - 1)}
 if #range_index_attr_keys > 0 then
   local attr_vals = redis.call('hmget', key, unpack(range_index_attr_keys))
   for i=1, #range_index_attr_keys do
-    delete_id_from_range_index_attr(model, range_index_attr_keys[i], attr_vals[i], id)
+    delete_id_from_range_index_attr(hash_tag, model, range_index_attr_keys[i], attr_vals[i], id)
   end
 end
+-- Delete record from custom indexes
+local custom_index_names = {unpack(ARGV, custom_index_pos)}
+for _, index_name in ipairs(custom_index_names) do
+  delete_record_from_custom_index(hash_tag, model, index_name, id)
+end
 
 -- delete the actual key
 return redis.call('del', key)

data/lib/redcord/server_scripts/find_by_attr.erb.lua

@@ -15,9 +15,11 @@ A hash of id:model of all the ids that match the query conditions given.
 -- accessed by Lua using the ARGV global variable, very similarly to what
 -- happens with keys (so ARGV[1], ARGV[2], ...).
 --
---   KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
---   ARGV[1...N] = attr_key [attr_key ..]
---
+--   KEYS[1] = hash_tag
+--   ARGV = Model.name custom_index_name num_index_attr num_range_index_attr num_custom_index_attr num_query_conditions 
+--          [index_attrs ...] [range_index_attrs ...] [custom_index_attrs ...] [query_conidtions ...] [attr_selections ...]
+--          [query_conidtions ...]: [attr_key1 attr_val1 attr_key2 attr_val2 ...]
+--          [attr_selections ...]: [attr_key1 attr_key2 ...]
 --   For equality query conditions, key value pairs are expected to appear in
 --   the KEYS array as [attr_key, attr_val]
 --   For range query conditions, key value pairs are expected to appear in the
@@ -29,27 +31,60 @@ A hash of id:model of all the ids that match the query conditions given.
 <%= include_lua 'shared/lua_helper_methods' %>
 <%= include_lua 'shared/query_helper_methods' %>
 
-if #KEYS < 3 then
-  error('Expected keys to be at least of size 3')
+if #KEYS ~=1 then
+  error('Expected keys to be of size 1')
 end
 
-local model = KEYS[1]
-local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
+local model = ARGV[1]
+
+local index_name = ARGV[2]
+local index_attr_pos = 7
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+local query_cond_pos = custom_attr_pos + ARGV[5]
+local attr_selection_pos = query_cond_pos + ARGV[6]
 
 -- Get all ids which have the corresponding attribute values.
 local ids_set = nil
--- For normal sets, Redis has SINTER built in to return the set intersection
-if #index_sets > 0 then
-   ids_set = to_set(redis.call('sinter', unpack(index_sets)))
-end
--- For sorted sets, call helper function zinter_zrangebyscore, which calls
--- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
-if #range_index_sets > 0 then
-  ids_set = intersect_range_index_sets(ids_set, range_index_sets)
+local index_sets, range_index_sets = {}, {}
+
+-- If custom index name is empty -> use single-attribute indices
+if index_name == '' then
+  index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(
+    KEYS[1],
+    model,
+    to_set({unpack(ARGV, index_attr_pos, range_attr_pos - 1)}),
+    to_set({unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}),
+    unpack(ARGV, query_cond_pos, attr_selection_pos - 1)
+  ))
+
+  -- For normal sets, Redis has SINTER built in to return the set intersection
+  if #index_sets > 0 then
+    ids_set = to_set(redis.call('sinter', unpack(index_sets)))
+  end
+  -- For sorted sets, call helper function zinter_zrangebyscore, which calls
+  -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
+  if #range_index_sets > 0 then
+    ids_set = intersect_range_index_sets(ids_set, range_index_sets)
+  end
+else
+  local custom_index_attrs = {unpack(ARGV, custom_attr_pos, query_cond_pos - 1)}
+  local custom_index_query = validate_and_parse_query_conditions_custom(
+    KEYS[1],
+    model,
+    index_name,
+    custom_index_attrs,
+    {unpack(ARGV, query_cond_pos, attr_selection_pos - 1)}
+  )
+  if #custom_index_query > 0 then
+    ids_set = get_id_set_from_custom_index(ids_set, custom_index_query)
+  else
+    ids_set = {}
+  end
 end
 
 -- Query for the hashes for all ids in the set intersection
-local res, stale_ids = unpack(batch_hget(model, ids_set, ARGV))
+local res, stale_ids = unpack(batch_hget(model, ids_set, unpack(ARGV, attr_selection_pos)))
 
 -- Delete any stale ids which are no longer in redis from the id sets.
 -- This can happen if an entry was auto expired due to ttl, but not removed up yet

data/lib/redcord/server_scripts/find_by_attr_count.erb.lua

@@ -15,7 +15,9 @@ An integer number of records that match the query conditions given.
 -- accessed by Lua using the ARGV global variable, very similarly to what
 -- happens with keys (so ARGV[1], ARGV[2], ...).
 --
---   KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
+--   KEYS[1] = hash_tag
+--   ARGV = Model.name custom_index_name num_index_attr num_range_index_attr num_custom_index_attr
+--          [index_attrs ...] [range_index_attrs ...] [custom_index_attrs ...] [query_conidtions ...]
 --
 --   For equality query conditions, key value pairs are expected to appear in
 --   the KEYS array as [attr_key, attr_val]
@@ -25,25 +27,54 @@ An integer number of records that match the query conditions given.
 <%= include_lua 'shared/lua_helper_methods' %>
 <%= include_lua 'shared/query_helper_methods' %>
 
-if #KEYS < 3 then
-  error('Expected keys to be at least of size 3')
+if #KEYS ~=1 then
+  error('Expected keys to be of size 1')
 end
 
-local model = KEYS[1]
-local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
+local model = ARGV[1]
+
+local index_name = ARGV[2]
+local index_attr_pos = 6
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+local query_cond_pos = custom_attr_pos + ARGV[5]
 
 -- Get all ids which have the corresponding attribute values.
 local ids_set = nil
--- For normal sets, Redis has SINTER built in to return the set intersection
-if #index_sets > 0 then
-   ids_set = to_set(redis.call('sinter', unpack(index_sets)))
-end
--- For sorted sets, call helper function zinter_zrangebyscore, which calls
--- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
-if #range_index_sets > 0 then
-  ids_set = intersect_range_index_sets(ids_set, range_index_sets)
-end
 
+if index_name == '' then
+  local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(
+    KEYS[1],
+    model,
+    to_set({unpack(ARGV, index_attr_pos, range_attr_pos - 1)}),
+    to_set({unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}),
+    unpack(ARGV, query_cond_pos)
+  ))
+
+  -- For normal sets, Redis has SINTER built in to return the set intersection
+  if #index_sets > 0 then
+    ids_set = to_set(redis.call('sinter', unpack(index_sets)))
+  end
+  -- For sorted sets, call helper function zinter_zrangebyscore, which calls
+  -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
+  if #range_index_sets > 0 then
+    ids_set = intersect_range_index_sets(ids_set, range_index_sets)
+  end
+else
+  local custom_index_attrs = {unpack(ARGV, custom_attr_pos, query_cond_pos - 1)}
+  local custom_index_query = validate_and_parse_query_conditions_custom(
+    KEYS[1],
+    model,
+    index_name,
+    custom_index_attrs,
+    {unpack(ARGV, query_cond_pos)}
+  )
+  if #custom_index_query > 0 then
+    ids_set = get_id_set_from_custom_index(ids_set, custom_index_query)
+  else
+    ids_set = {}
+  end
+end
 -- Get the number of records which satisfy the query conditions.
 -- We do not delete stale ids as part of this function call because we do not have
 -- the list of ids which don't exist. The Redis command EXISTS key [key ...] is an O(1)

data/lib/redcord/server_scripts/shared/index_helper_methods.erb.lua

@@ -1,61 +1,90 @@
 -- Add an id to the id set of the index attribute
-local function add_id_to_index_attr(model, attr_key, attr_val, id)
+local function add_id_to_index_attr(hash_tag, model, attr_key, attr_val, id)
   if attr_val then
     -- Call the Redis command: SADD "#{Model.name}:#{attr_name}:#{attr_val}" member ..
-    redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val, id)
+    redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag, id)
   end
 end
 
 -- Remove an id from the id set of the index attribute
-local function delete_id_from_index_attr(model, attr_key, attr_val, id)
+local function delete_id_from_index_attr(hash_tag, model, attr_key, attr_val, id)
   if attr_val then
     -- Call the Redis command: SREM "#{Model.name}:#{attr_name}:#{attr_val}" member ..
-    redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val, id)
+    redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag, id)
   end
 end
 
 -- Move an id from one id set to another for the index attribute
-local function replace_id_in_index_attr(model, attr_key, prev_attr_val,curr_attr_val, id)
+local function replace_id_in_index_attr(hash_tag, model, attr_key, prev_attr_val,curr_attr_val, id)
   -- If previous and new value differs, then modify the id sets accordingly
   if prev_attr_val ~= curr_attr_val then
-    delete_id_from_index_attr(model, attr_key, prev_attr_val, id)
-    add_id_to_index_attr(model, attr_key, curr_attr_val, id)
+    delete_id_from_index_attr(hash_tag, model, attr_key, prev_attr_val, id)
+    add_id_to_index_attr(hash_tag, model, attr_key, curr_attr_val, id)
   end
 end
 
 -- Add an id to the sorted id set of the range index attribute
-local function add_id_to_range_index_attr(model, attr_key, attr_val, id)
+local function add_id_to_range_index_attr(hash_tag, model, attr_key, attr_val, id)
   if attr_val then
     -- Nil values of range indices are sent to Redis as an empty string. They are stored
     -- as a regular set at key "#{Model.name}:#{attr_name}:"
     if attr_val == "" then
-      redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val, id)
+      redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag, id)
     else
       -- Call the Redis command: ZADD "#{Model.name}:#{attr_name}" #{attr_val} member ..,
       -- where attr_val is the score of the sorted set
-      redis.call('zadd', model .. ':' .. attr_key, attr_val, id)
+      redis.call('zadd', model .. ':' .. attr_key .. hash_tag, attr_val, id)
     end
   end
 end
 
 -- Remove an id from the sorted id set of the range index attribute
-local function delete_id_from_range_index_attr(model, attr_key, attr_val, id)
+local function delete_id_from_range_index_attr(hash_tag, model, attr_key, attr_val, id)
   if attr_val then
     -- Nil values of range indices are sent to Redis as an empty string. They are stored
     -- as a regular set at key "#{Model.name}:#{attr_name}:"
     if attr_val == "" then
-      redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val, id)
+      redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag, id)
     else
       -- Call the Redis command: ZREM "#{Model.name}:#{attr_name}:#{attr_val}" member ..
-      redis.call('zrem', model .. ':' .. attr_key, id)
+      redis.call('zrem', model .. ':' .. attr_key .. hash_tag, id)
     end
   end
 end
 
 -- Move an id from one sorted id set to another for the range index attribute
-local function replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+local function replace_id_in_range_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
   if prev_attr_val ~= curr_attr_val then
-    delete_id_from_range_index_attr(model, attr_key, prev_attr_val, id)
-    add_id_to_range_index_attr(model, attr_key, curr_attr_val, id)
+    delete_id_from_range_index_attr(hash_tag, model, attr_key, prev_attr_val, id)
+    add_id_to_range_index_attr(hash_tag, model, attr_key, curr_attr_val, id)
+  end
+end
+
+-- Add an index record to the sorted set of the custom index
+local function add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
+  local sep = ':'
+  if attr_values then
+    local index_string = ''
+    local attr_value_string = ''
+    for i, attr_value in ipairs(attr_values) do
+      if i > 1 then
+        index_string = index_string .. sep
+      end
+      attr_value_string = adjust_string_length(attr_value)
+      index_string = index_string .. attr_value_string
+    end
+    redis.call('zadd', model .. sep .. 'custom_index' .. sep .. index_name .. hash_tag, 0, index_string .. sep .. id)
+    redis.call('hset', model .. sep .. 'custom_index' .. sep .. index_name .. '_content' .. hash_tag, id, index_string .. sep .. id)
+  end
+end
+
+-- Remove a record from the sorted set of the custom index
+local function delete_record_from_custom_index(hash_tag, model, index_name, id)
+  local sep = ':'
+  local index_key = model .. sep .. 'custom_index' .. sep .. index_name
+  local index_string = redis.call('hget', index_key .. '_content' .. hash_tag, id)
+  if index_string then
+    redis.call('zremrangebylex', index_key .. hash_tag, '[' .. index_string, '[' .. index_string)
+    redis.call('hdel', index_key .. '_content' .. hash_tag, id)
   end
 end

data/lib/redcord/server_scripts/shared/lua_helper_methods.erb.lua

@@ -1,9 +1,8 @@
 -- Helper function to convert argument array to hash set
-local function to_hash(list)
+local function to_hash(...)
   local hash = {}
-  if not list then return hash end
-  for i=1, #list, 2 do
-    hash[list[i]] = list[i+1]
+  for i=1, #arg, 2 do
+    hash[arg[i]] = arg[i+1]
   end
   return hash
 end
@@ -31,3 +30,20 @@ local function set_list_intersect(set, list)
   end
   return set_intersect
 end
+
+-- Helper function to transform attribute values so that they become comparable as strings
+local function adjust_string_length(value)
+  if value == '' or value == nil then
+    return '!'
+  end
+  if string.sub(value, 1, 1) == '-' then
+    error("Custom index currently doesn't support negative values")
+  end
+  local whole_digits_count = 19
+  local res = string.rep('0', whole_digits_count - string.len(value)) .. value
+  if string.len(res) > whole_digits_count then
+    error("Custom index can't be used if string representation of whole part of attribute value is longer than " .. 
+           whole_digits_count .. ' characters')
+  end
+  return res
+end

data/lib/redcord/server_scripts/shared/query_helper_methods.erb.lua

@@ -10,18 +10,37 @@ local function intersect_range_index_sets(set, tuples)
   return set
 end
 
+-- Runs a query against a sorted set, extracts ids.
+-- Response from redis: attr_value:[attr_value ...]:id
+-- Returns a set of ids.
+local function get_id_set_from_custom_index(set, query)
+  local ids = {}
+  local index_strings = {}
+  local sep = ':'
+  local id = ''
+  local key, min, max = unpack(query)
+  index_strings = redis.call('zrangebylex', key, min, max)
+  for _, index_string in ipairs(index_strings) do
+    id = string.match(index_string, '[^' .. sep .. ']+$')
+    table.insert(ids, id)
+  end
+  set = to_set(ids)
+  
+  return set
+end
+
 -- Gets the hash of all the ids given. Returns the results in a
 -- table, as well as any ids not found in Redis as a separate table
-local function batch_hget(model, ids_set, fields)
+local function batch_hget(model, ids_set, ...)
   local res, stale_ids = {}, {}
   for id, _ in pairs(ids_set) do
     local instance = nil
-    if fields and #fields > 0 then
-      local values = redis.call('hmget', model .. ':id:' .. id, unpack(fields))
+    if #{...}> 0 then
+      local values = redis.call('hmget', model .. ':id:' .. id, ...)
       -- HMGET returns the value in the order of the fields given. Map back to
       -- field value [field value ..]
       instance = {}
-      for i, field in ipairs(fields) do
+      for i, field in ipairs({...}) do
         if not values[i] then
           instance = nil
           break
@@ -76,30 +95,28 @@ end
 -- attributes. Parse query conditions into two separate tables: 
 -- 1. index_sets formatted as the id set keys in Redis '#{Model.name}:#{attr_key}:#{attr_val}'
 -- 2. range_index_sets formatted as a tuple {id set key, min, max} => { '#{Model.name}:#{attr_key}' min max }
-local function validate_and_parse_query_conditions(model, args)
-  local index_attrs = to_set(redis.call('smembers', model .. ':index_attrs'))
-  local range_index_attrs = to_set(redis.call('smembers', model .. ':range_index_attrs'))
+local function validate_and_parse_query_conditions(hash_tag, model, index_attrs, range_index_attrs, ...) 
   -- Iterate through the arguments of the script to form the redis keys at which the
   -- indexed id sets are stored.
   local index_sets, range_index_sets = {}, {}
-  local i = 2
-  while i <= #args do
-    local attr_key, attr_val = args[i], args[i+1]
+  local i = 1
+  while i <= #arg do
+    local attr_key, attr_val = arg[i], arg[i+1]
     if index_attrs[attr_key] then
       validate_attr_vals(attr_key, {attr_val})
       -- For normal index attributes, keys are stored at "#{Model.name}:#{attr_key}:#{attr_val}"
-      table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val)
+      table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag)
       i = i + 2
     elseif range_index_attrs[attr_key] then
       -- For range attributes, nil values are stored as normal sets
       if attr_val == "" then
-        table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val)
+        table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag)
         i = i + 2
       else
-        local min, max = args[i+1], args[i+2]
+        local min, max = arg[i+1], arg[i+2]
         validate_attr_vals(attr_key, {min, max})
         -- For range index attributes, they are stored at "#{Model.name}:#{attr_key}"
-        table.insert(range_index_sets, {model .. ':' .. attr_key, min, max})
+        table.insert(range_index_sets, {model .. ':' .. attr_key .. hash_tag, min, max})
         i = i + 3
       end
     else
@@ -108,3 +125,53 @@ local function validate_and_parse_query_conditions(model, args)
   end
   return {index_sets, range_index_sets}
 end
+
+-- Validates that attributes in query are in correct order and range condition is applied only on the last attribute.
+-- '~' is used as a character that is lexicographically greater than any alphanumerical. '['  makes range inclusive (exclusive are not yet supported)
+-- Returns a table {index_key, min_string, max_string} to be used for index query.
+local function validate_and_parse_query_conditions_custom(hash_tag, model, index_name, custom_index_attrs, args)
+  if #custom_index_attrs == 0 then
+    error('Index ' .. index_name .. ' does not exist')
+  end
+  local sep = ':'
+  local i = 1
+  local j = 1
+  local min, value_string_min, query_string_min = '', '', ''
+  local max, value_string_max, query_string_max = '', '', ''
+  local is_prev_attr_query_range = false
+  while i <= #args do
+    if is_prev_attr_query_range then
+      error('Range can be applied to the last attribute of query only')
+    end  
+    local attr_key = args[i]
+    if custom_index_attrs[j] == attr_key then
+      min, max = args[i+1], args[i+2]
+      if j > 1 then
+        query_string_min = query_string_min .. sep
+        query_string_max = query_string_max .. sep
+      else
+        query_string_min = query_string_min .. '['
+        query_string_max = query_string_max .. '['
+      end
+      if min ~= '-inf' then
+        value_string_min = adjust_string_length(min)
+        query_string_min = query_string_min .. value_string_min
+      end
+      if max ~= '+inf' then
+        value_string_max = adjust_string_length(max)
+        query_string_max = query_string_max .. value_string_max
+      else
+        query_string_max = query_string_max .. '~'
+      end
+      if min ~= max then
+        is_prev_attr_query_range = true
+      end
+      j = j + 1
+      i = i + 3
+    else
+      error(attr_key .. ' in position ' .. j .. ' is not supported by index ' .. index_name)
+    end
+  end
+  query_string_max = query_string_max .. sep .. '~'
+  return {model .. sep .. 'custom_index' .. sep .. index_name .. hash_tag, query_string_min, query_string_max}
+end