redcord 0.0.3 → 0.1.3

data/lib/redcord/server_scripts/shared/index_helper_methods.erb.lua

@@ -1,61 +1,90 @@
 -- Add an id to the id set of the index attribute
-local function add_id_to_index_attr(model, attr_key, attr_val, id)
+local function add_id_to_index_attr(hash_tag, model, attr_key, attr_val, id)
   if attr_val then
     -- Call the Redis command: SADD "#{Model.name}:#{attr_name}:#{attr_val}" member ..
-    redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val, id)
+    redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag, id)
   end
 end
 
 -- Remove an id from the id set of the index attribute
-local function delete_id_from_index_attr(model, attr_key, attr_val, id)
+local function delete_id_from_index_attr(hash_tag, model, attr_key, attr_val, id)
   if attr_val then
     -- Call the Redis command: SREM "#{Model.name}:#{attr_name}:#{attr_val}" member ..
-    redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val, id)
+    redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag, id)
   end
 end
 
 -- Move an id from one id set to another for the index attribute
-local function replace_id_in_index_attr(model, attr_key, prev_attr_val,curr_attr_val, id)
+local function replace_id_in_index_attr(hash_tag, model, attr_key, prev_attr_val,curr_attr_val, id)
   -- If previous and new value differs, then modify the id sets accordingly
   if prev_attr_val ~= curr_attr_val then
-    delete_id_from_index_attr(model, attr_key, prev_attr_val, id)
-    add_id_to_index_attr(model, attr_key, curr_attr_val, id)
+    delete_id_from_index_attr(hash_tag, model, attr_key, prev_attr_val, id)
+    add_id_to_index_attr(hash_tag, model, attr_key, curr_attr_val, id)
   end
 end
 
 -- Add an id to the sorted id set of the range index attribute
-local function add_id_to_range_index_attr(model, attr_key, attr_val, id)
+local function add_id_to_range_index_attr(hash_tag, model, attr_key, attr_val, id)
   if attr_val then
     -- Nil values of range indices are sent to Redis as an empty string. They are stored
     -- as a regular set at key "#{Model.name}:#{attr_name}:"
     if attr_val == "" then
-      redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val, id)
+      redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag, id)
     else
       -- Call the Redis command: ZADD "#{Model.name}:#{attr_name}" #{attr_val} member ..,
       -- where attr_val is the score of the sorted set
-      redis.call('zadd', model .. ':' .. attr_key, attr_val, id)
+      redis.call('zadd', model .. ':' .. attr_key .. hash_tag, attr_val, id)
     end
   end
 end
 
 -- Remove an id from the sorted id set of the range index attribute
-local function delete_id_from_range_index_attr(model, attr_key, attr_val, id)
+local function delete_id_from_range_index_attr(hash_tag, model, attr_key, attr_val, id)
   if attr_val then
     -- Nil values of range indices are sent to Redis as an empty string. They are stored
     -- as a regular set at key "#{Model.name}:#{attr_name}:"
     if attr_val == "" then
-      redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val, id)
+      redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag, id)
     else
       -- Call the Redis command: ZREM "#{Model.name}:#{attr_name}:#{attr_val}" member ..
-      redis.call('zrem', model .. ':' .. attr_key, id)
+      redis.call('zrem', model .. ':' .. attr_key .. hash_tag, id)
     end
   end
 end
 
 -- Move an id from one sorted id set to another for the range index attribute
-local function replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+local function replace_id_in_range_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
   if prev_attr_val ~= curr_attr_val then
-    delete_id_from_range_index_attr(model, attr_key, prev_attr_val, id)
-    add_id_to_range_index_attr(model, attr_key, curr_attr_val, id)
+    delete_id_from_range_index_attr(hash_tag, model, attr_key, prev_attr_val, id)
+    add_id_to_range_index_attr(hash_tag, model, attr_key, curr_attr_val, id)
+  end
+end
+
+-- Add an index record to the sorted set of the custom index
+local function add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
+  local sep = ':'
+  if attr_values then
+    local index_string = ''
+    local attr_value_string = ''
+    for i, attr_value in ipairs(attr_values) do
+      if i > 1 then
+        index_string = index_string .. sep
+      end
+      attr_value_string = adjust_string_length(attr_value)
+      index_string = index_string .. attr_value_string
+    end
+    redis.call('zadd', model .. sep .. 'custom_index' .. sep .. index_name .. hash_tag, 0, index_string .. sep .. id)
+    redis.call('hset', model .. sep .. 'custom_index' .. sep .. index_name .. '_content' .. hash_tag, id, index_string .. sep .. id)
+  end
+end
+
+-- Remove a record from the sorted set of the custom index
+local function delete_record_from_custom_index(hash_tag, model, index_name, id)
+  local sep = ':'
+  local index_key = model .. sep .. 'custom_index' .. sep .. index_name
+  local index_string = redis.call('hget', index_key .. '_content' .. hash_tag, id)
+  if index_string then
+    redis.call('zremrangebylex', index_key .. hash_tag, '[' .. index_string, '[' .. index_string)
+    redis.call('hdel', index_key .. '_content' .. hash_tag, id)
   end
 end

data/lib/redcord/server_scripts/shared/lua_helper_methods.erb.lua

@@ -1,9 +1,8 @@
 -- Helper function to convert argument array to hash set
-local function to_hash(list)
+local function to_hash(...)
   local hash = {}
-  if not list then return hash end
-  for i=1, #list, 2 do
-    hash[list[i]] = list[i+1]
+  for i=1, #arg, 2 do
+    hash[arg[i]] = arg[i+1]
   end
   return hash
 end
@@ -31,3 +30,20 @@ local function set_list_intersect(set, list)
   end
   return set_intersect
 end
+
+-- Helper function to transform attribute values so that they become comparable as strings
+local function adjust_string_length(value)
+  if value == '' or value == nil then
+    return '!'
+  end
+  if string.sub(value, 1, 1) == '-' then
+    error("Custom index currently doesn't support negative values")
+  end
+  local whole_digits_count = 19
+  local res = string.rep('0', whole_digits_count - string.len(value)) .. value
+  if string.len(res) > whole_digits_count then
+    error("Custom index can't be used if string representation of whole part of attribute value is longer than " .. 
+           whole_digits_count .. ' characters')
+  end
+  return res
+end

data/lib/redcord/server_scripts/shared/query_helper_methods.erb.lua

@@ -10,18 +10,37 @@ local function intersect_range_index_sets(set, tuples)
   return set
 end
 
+-- Runs a query against a sorted set, extracts ids.
+-- Response from redis: attr_value:[attr_value ...]:id
+-- Returns a set of ids.
+local function get_id_set_from_custom_index(set, query)
+  local ids = {}
+  local index_strings = {}
+  local sep = ':'
+  local id = ''
+  local key, min, max = unpack(query)
+  index_strings = redis.call('zrangebylex', key, min, max)
+  for _, index_string in ipairs(index_strings) do
+    id = string.match(index_string, '[^' .. sep .. ']+$')
+    table.insert(ids, id)
+  end
+  set = to_set(ids)
+  
+  return set
+end
+
 -- Gets the hash of all the ids given. Returns the results in a
 -- table, as well as any ids not found in Redis as a separate table
-local function batch_hget(model, ids_set, fields)
+local function batch_hget(model, ids_set, ...)
   local res, stale_ids = {}, {}
   for id, _ in pairs(ids_set) do
     local instance = nil
-    if fields and #fields > 0 then
-      local values = redis.call('hmget', model .. ':id:' .. id, unpack(fields))
+    if #{...}> 0 then
+      local values = redis.call('hmget', model .. ':id:' .. id, ...)
       -- HMGET returns the value in the order of the fields given. Map back to
       -- field value [field value ..]
       instance = {}
-      for i, field in ipairs(fields) do
+      for i, field in ipairs({...}) do
         if not values[i] then
           instance = nil
           break
@@ -76,30 +95,28 @@ end
 -- attributes. Parse query conditions into two separate tables: 
 -- 1. index_sets formatted as the id set keys in Redis '#{Model.name}:#{attr_key}:#{attr_val}'
 -- 2. range_index_sets formatted as a tuple {id set key, min, max} => { '#{Model.name}:#{attr_key}' min max }
-local function validate_and_parse_query_conditions(model, args)
-  local index_attrs = to_set(redis.call('smembers', model .. ':index_attrs'))
-  local range_index_attrs = to_set(redis.call('smembers', model .. ':range_index_attrs'))
+local function validate_and_parse_query_conditions(hash_tag, model, index_attrs, range_index_attrs, ...) 
   -- Iterate through the arguments of the script to form the redis keys at which the
   -- indexed id sets are stored.
   local index_sets, range_index_sets = {}, {}
-  local i = 2
-  while i <= #args do
-    local attr_key, attr_val = args[i], args[i+1]
+  local i = 1
+  while i <= #arg do
+    local attr_key, attr_val = arg[i], arg[i+1]
     if index_attrs[attr_key] then
       validate_attr_vals(attr_key, {attr_val})
       -- For normal index attributes, keys are stored at "#{Model.name}:#{attr_key}:#{attr_val}"
-      table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val)
+      table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag)
       i = i + 2
     elseif range_index_attrs[attr_key] then
       -- For range attributes, nil values are stored as normal sets
       if attr_val == "" then
-        table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val)
+        table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val .. hash_tag)
         i = i + 2
       else
-        local min, max = args[i+1], args[i+2]
+        local min, max = arg[i+1], arg[i+2]
         validate_attr_vals(attr_key, {min, max})
         -- For range index attributes, they are stored at "#{Model.name}:#{attr_key}"
-        table.insert(range_index_sets, {model .. ':' .. attr_key, min, max})
+        table.insert(range_index_sets, {model .. ':' .. attr_key .. hash_tag, min, max})
         i = i + 3
       end
     else
@@ -108,3 +125,53 @@ local function validate_and_parse_query_conditions(model, args)
   end
   return {index_sets, range_index_sets}
 end
+
+-- Validates that attributes in query are in correct order and range condition is applied only on the last attribute.
+-- '~' is used as a character that is lexicographically greater than any alphanumerical. '['  makes range inclusive (exclusive are not yet supported)
+-- Returns a table {index_key, min_string, max_string} to be used for index query.
+local function validate_and_parse_query_conditions_custom(hash_tag, model, index_name, custom_index_attrs, args)
+  if #custom_index_attrs == 0 then
+    error('Index ' .. index_name .. ' does not exist')
+  end
+  local sep = ':'
+  local i = 1
+  local j = 1
+  local min, value_string_min, query_string_min = '', '', ''
+  local max, value_string_max, query_string_max = '', '', ''
+  local is_prev_attr_query_range = false
+  while i <= #args do
+    if is_prev_attr_query_range then
+      error('Range can be applied to the last attribute of query only')
+    end  
+    local attr_key = args[i]
+    if custom_index_attrs[j] == attr_key then
+      min, max = args[i+1], args[i+2]
+      if j > 1 then
+        query_string_min = query_string_min .. sep
+        query_string_max = query_string_max .. sep
+      else
+        query_string_min = query_string_min .. '['
+        query_string_max = query_string_max .. '['
+      end
+      if min ~= '-inf' then
+        value_string_min = adjust_string_length(min)
+        query_string_min = query_string_min .. value_string_min
+      end
+      if max ~= '+inf' then
+        value_string_max = adjust_string_length(max)
+        query_string_max = query_string_max .. value_string_max
+      else
+        query_string_max = query_string_max .. '~'
+      end
+      if min ~= max then
+        is_prev_attr_query_range = true
+      end
+      j = j + 1
+      i = i + 3
+    else
+      error(attr_key .. ' in position ' .. j .. ' is not supported by index ' .. index_name)
+    end
+  end
+  query_string_max = query_string_max .. sep .. '~'
+  return {model .. sep .. 'custom_index' .. sep .. index_name .. hash_tag, query_string_min, query_string_max}
+end

data/lib/redcord/server_scripts/update_hash.erb.lua

@@ -16,21 +16,24 @@ nil
 -- accessed by Lua using the ARGV global variable, very similarly to what
 -- happens with keys (so ARGV[1], ARGV[2], ...).
 --
---   KEYS[1] = redcord_instance.class.name
---   KEYS[2] = redcord_instance.id
---   ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
+--   KEYS = redcord_instance.id hash_tag
+--   ARGV = Model.name ttl index_attr_size range_index_attr_size custom_index_attrs_flat_size [index_attr_key ...] [range_index_attr_key ...]
+--          [custom_index_name attrs_size [custom_index_attr_key ...] ...] attr_key attr_val [attr_key attr_val ..]
 <%= include_lua 'shared/lua_helper_methods' %>
 <%= include_lua 'shared/index_helper_methods' %>
 
 if #KEYS ~= 2 then
   error('Expected keys of be of size 2')
 end
-if #ARGV % 2 ~= 0 then
-  error('Expected an even number of arguments')
-end
 
-local model = KEYS[1]
-local id = KEYS[2]
+local model, ttl = unpack(ARGV)
+local id, hash_tag = unpack(KEYS)
+
+local index_attr_pos = 6
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+-- Starting position of the attr_key-attr_val pairs
+local attr_pos = custom_attr_pos + ARGV[5]
 
 -- key = "#{model}:id:{id}"
 local key = model .. ':id:' .. id
@@ -44,8 +47,8 @@ if redis.call('exists', key) == 0 then
 end
 
 -- Modify the id sets for any indexed attributes
-local attrs_hash = to_hash(ARGV)
-local indexed_attr_keys = redis.call('smembers', model .. ':index_attrs')
+local attrs_hash = to_hash(unpack(ARGV, attr_pos))
+local indexed_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
 if #indexed_attr_keys > 0 then
   -- Get the previous and new values for indexed attributes
   local prev_attrs = redis.call('hmget', key, unpack(indexed_attr_keys))
@@ -53,11 +56,11 @@ if #indexed_attr_keys > 0 then
     local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
     -- Skip attr values not present in the argument hash
     if curr_attr_val then
-      replace_id_in_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+      replace_id_in_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
     end
   end
 end
-local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
+local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}
 if #range_index_attr_keys > 0 then
   -- Get the previous and new values for indexed attributes
   local prev_attrs = redis.call('hmget', key, unpack(range_index_attr_keys))
@@ -65,27 +68,38 @@ if #range_index_attr_keys > 0 then
     local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
     -- Skip attr values not present in the argument hash
     if curr_attr_val then
-      replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+      replace_id_in_range_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
     end
   end
 end
 
 -- Forward the script arguments to the Redis command HSET and update the args.
 -- Call the Redis command: HSET key [field value ...]
-redis.call('hset', key, unpack(ARGV))
-
--- Call the Redis command: GET "#{Model.name}:ttl"
-local ttl = redis.call('get', model .. ':ttl')
+redis.call('hset', key, unpack(ARGV, attr_pos))
 
-if ttl then
-  if ttl == '-1' then
-    -- Persist the object if the ttl is set to -1
-    redis.call('persist', key)
-  else
-    -- Reset the TTL for this object. We do this manually becaues altering the
-    -- field value of a hash with HSET, etc. will leave the TTL
-    -- untouched: https://redis.io/commands/expire
-    redis.call('expire', key, ttl)
+-- Update custom indexes
+local updated_hash = to_hash(unpack(redis.call('hgetall', key)))
+local custom_index_attr_keys = {unpack(ARGV, custom_attr_pos, attr_pos - 1)}
+local i = 1
+while i < #custom_index_attr_keys do
+  local index_name, attrs_num = custom_index_attr_keys[i], custom_index_attr_keys[i+1]
+  local attr_values = {}
+  for j, attr_key in ipairs({unpack(custom_index_attr_keys, i + 2, i + attrs_num + 1)}) do
+    attr_values[j] = updated_hash[attr_key]
   end
+  delete_record_from_custom_index(hash_tag, model, index_name, id)
+  add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
+  i = i + 2 + attrs_num
+end
+
+-- Call the Redis command: GET "#{Model.name}:ttl"
+if ttl == '-1' then
+  -- Persist the object if the ttl is set to -1
+  redis.call('persist', key)
+else
+  -- Reset the TTL for this object. We do this manually becaues altering the
+  -- field value of a hash with HSET, etc. will leave the TTL
+  -- untouched: https://redis.io/commands/expire
+  redis.call('expire', key, ttl)
 end
 return nil

data/lib/redcord/tasks/redis.rake

@@ -1,6 +1,7 @@
 # typed: strict
 require 'redcord/migration/version'
 require 'redcord/migration/migrator'
+require 'redcord/vacuum_helper'
 
 db_namespace = namespace :redis do
   task migrate: :environment do
@@ -31,4 +32,18 @@ db_namespace = namespace :redis do
     migration_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     puts "\nFinished in #{(migration_end - migration_start).round(3)} seconds"
   end
+
+  task :vacuum, [:model_name] => :environment do |t, args|
+    desc "Vacuum index attributes for stale ids on a Redcord model"
+    $stdout.sync = true
+    model_name = args[:model_name]
+    puts "Attempting to vacuum the index attributes of the Redcord model: #{model_name}"
+    vacuum_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    Redcord::VacuumHelper.vacuum(Object.const_get(args[:model_name]))
+    vacuum_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    puts "Finished vacuuming #{model_name} in #{(vacuum_end - vacuum_start).round(3)} seconds"
+  rescue NameError => e
+    raise StandardError.new("#{args[:model_name]} is not a valid Redcord model.")
+  end
 end

data/lib/redcord/vacuum_helper.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+module Redcord::VacuumHelper
+  extend T::Sig
+  extend T::Helpers
+
+  sig { params(model: T.class_of(Redcord::Base)).void }
+  def self.vacuum(model)
+    model.class_variable_get(:@@index_attributes).each do |index_attr|
+      puts "Vacuuming index attribute: #{index_attr}"
+      _vacuum_index_attribute(model, index_attr)
+    end
+    model.class_variable_get(:@@range_index_attributes).each do |range_index_attr|
+      puts "Vacuuming range index attribute: #{range_index_attr}"
+      _vacuum_range_index_attribute(model, range_index_attr)
+    end
+    model.class_variable_get(:@@custom_index_attributes).keys.each do |index_name|
+      puts "Vacuuming custom index: #{index_name}"
+      _vacuum_custom_index(model, index_name)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), index_attr: Symbol).void }
+  def self._vacuum_index_attribute(model, index_attr)
+    # Scan through all index attribute values by matching on Redcord:Model:index_attr:*
+    model.redis.scan_each_shard("#{model.model_key}:#{index_attr}:*") do |key|
+      _remove_stale_ids_from_set(model, key)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), range_index_attr: Symbol).void }
+  def self._vacuum_range_index_attribute(model, range_index_attr)
+    key_suffix = model.shard_by_attribute.nil? ? nil : '{*}'
+    range_index_set_key = "#{model.model_key}:#{range_index_attr}"
+    range_index_set_nil_key = "#{range_index_set_key}:"
+
+    # Handle nil values for range index attributes, which are stored in a normal
+    # set at Redcord:Model:range_index_attr:
+    model.redis.scan_each_shard("#{range_index_set_nil_key}#{key_suffix}") do |key|
+      _remove_stale_ids_from_set(model, key)
+    end
+
+    model.redis.scan_each_shard("#{range_index_set_key}#{key_suffix}") do |key|
+      _remove_stale_ids_from_sorted_set(model, key)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
+  def self._vacuum_custom_index(model, index_name)
+    key_suffix = model.shard_by_attribute.nil? ? nil : '{*}'
+    custom_index_content_key = "#{model.model_key}:custom_index:#{index_name}_content"
+
+    model.redis.scan_each_shard("#{custom_index_content_key}#{key_suffix}") do |key|
+      hash_tag = key.split(custom_index_content_key)[1] || ""
+      _remove_stale_records_from_custom_index(model, hash_tag, index_name)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), set_key: String).void }
+  def self._remove_stale_ids_from_set(model, set_key)
+    model.redis.sscan_each(set_key) do |id|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.srem(set_key, id)
+      end
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), sorted_set_key: String).void }
+  def self._remove_stale_ids_from_sorted_set(model, sorted_set_key)
+    model.redis.zscan_each(sorted_set_key) do |id, _|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.zrem(sorted_set_key, id)
+      end
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), hash_tag: String, index_name: Symbol).void }
+  def self._remove_stale_records_from_custom_index(model, hash_tag, index_name)
+    index_key = "#{model.model_key}:custom_index:#{index_name}#{hash_tag}"
+    index_content_key = "#{model.model_key}:custom_index:#{index_name}_content#{hash_tag}"
+    model.redis.hscan_each(index_content_key).each do |id, index_string|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.hdel(index_content_key, id)
+        model.redis.zremrangebylex(index_key, "[#{index_string}", "[#{index_string}")
+      end
+    end
+  end
+end