redcord 0.0.1.alpha

data/lib/redcord/serializer.rb

@@ -0,0 +1,129 @@
+require 'redcord/range_interval'
+# typed: strict
+#
+# This module defines various helper methods on Redcord for serialization between the 
+# Ruby client and Redis server.
+module Redcord
+  # Raised by Model.where
+  class AttributeNotIndexed < StandardError; end
+  class WrongAttributeType < TypeError; end
+end
+
+module Redcord::Serializer
+  extend T::Sig
+
+  sig { params(klass: T.any(Module, T.class_of(T::Struct))).void }
+  def self.included(klass)
+    klass.extend(ClassMethods)
+  end
+  
+  module ClassMethods
+    extend T::Sig
+
+    # Redis only allows range queries on floats. To allow range queries on the Ruby Time
+    # type, encode_attr_value and decode_attr_value will implicitly encode and decode
+    # Time attributes to a float.
+    TIME_TYPES = T.let(Set[Time, T.nilable(Time)], T::Set[T.untyped])
+    sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
+    def encode_attr_value(attribute, val)
+      if val && TIME_TYPES.include?(props[attribute][:type])
+        val = val.to_f
+      end
+      val
+    end
+
+    sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
+    def decode_attr_value(attribute, val)
+      if val && TIME_TYPES.include?(props[attribute][:type])
+        val = Time.zone.at(val.to_f)
+      end
+      val
+    end
+
+    sig { params(attr_key: Symbol, attr_val: T.untyped).returns(T.untyped)}
+    def validate_and_encode_query(attr_key, attr_val)
+      # Validate that attributes queried for are index attributes
+      if !class_variable_get(:@@index_attributes).include?(attr_key) &&
+        !class_variable_get(:@@range_index_attributes).include?(attr_key)
+        raise Redcord::AttributeNotIndexed.new(
+        "#{attr_key} is not an indexed attribute."
+        )
+      end
+      # Validate attribute types for normal index attributes
+      attr_type = get_attr_type(attr_key)
+      if class_variable_get(:@@index_attributes).include?(attr_key)
+        validate_attr_type(attr_val, attr_type)
+      else
+        # Validate attribute types for range index attributes
+        if attr_val.is_a?(Redcord::RangeInterval)
+          validate_attr_type(attr_val.min, T.cast(T.nilable(attr_type), T::Types::Base))
+          validate_attr_type(attr_val.max, T.cast(T.nilable(attr_type), T::Types::Base))
+        else
+          validate_attr_type(attr_val, attr_type)
+        end
+        # Range index attributes need to be further encoded into a format understood by the Lua script.
+        if attr_val != nil
+          attr_val = encode_range_index_attr_val(attr_key, attr_val)
+        end
+      end
+      attr_val
+    end
+
+    sig { params(attr_val: T.untyped, attr_type: T.any(Class, T::Types::Base)).void }
+    def validate_attr_type(attr_val, attr_type)
+      if (attr_type.is_a?(Class) && !attr_val.is_a?(attr_type)) ||
+        (attr_type.is_a?(T::Types::Base) && !attr_type.valid?(attr_val))
+        raise Redcord::WrongAttributeType.new(
+          "Expected type #{attr_type}, got #{attr_val.class}"
+        )
+      end
+    end
+
+    sig { params(attribute: Symbol, val: T.untyped).returns([T.untyped, T.untyped]) }
+    def encode_range_index_attr_val(attribute, val)
+      if val.is_a?(Redcord::RangeInterval)
+        # nil is treated as -inf and +inf. This is supported in Redis sorted sets
+        # so clients aren't required to know the highest and lowest scores in a range
+        min_val = !val.min ? '-inf' : encode_attr_value(attribute, val.min)
+        max_val = !val.max ? '+inf' : encode_attr_value(attribute, val.max)
+
+        # In Redis, by default min and max is closed. You can prefix the score with '(' to
+        # specify an open interval.
+        min_val = val.min_exclusive ? '(' + min_val.to_s : min_val.to_s
+        max_val = val.max_exclusive ? '(' + max_val.to_s : max_val.to_s
+        return [min_val, max_val]
+      else
+        # Equality queries for range indices are be passed to redis as a range [val, val].
+        encoded_val = encode_attr_value(attribute, val)
+        [encoded_val, encoded_val]
+      end
+    end
+
+    sig { params(attr_key: Symbol).returns(T.any(Class, T::Types::Base)) }
+    def get_attr_type(attr_key)
+      props[attr_key][:type_object]
+    end
+
+    sig { params(redis_hash: T::Hash[T.untyped, T.untyped], id: Integer).returns(T.untyped) }
+    def coerce_and_set_id(redis_hash, id)
+      # Coerce each serialized result returned from Redis back into Model instance
+      instance = TypeCoerce.send(:[], self).new.from(from_redis_hash(redis_hash))
+      instance.send(:id=, id)
+      instance
+    end
+    sig { returns(String) }
+    def model_key
+      "Redcord:#{name}"
+    end
+
+    sig { params(args: T::Hash[T.any(String, Symbol), T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
+    def to_redis_hash(args)
+      args.map { |key, val| [key.to_sym, encode_attr_value(key.to_sym, val)] }.to_h
+    end
+
+    sig { params(args: T::Hash[T.untyped, T.untyped]).returns(T::Hash[T.untyped, T.untyped]) }
+    def from_redis_hash(args)
+      args.map { |key, val| [key, decode_attr_value(key.to_sym, val)] }.to_h
+    end
+  end
+end

data/lib/redcord/server_scripts.rb

@@ -0,0 +1,78 @@
+# typed: strict
+module Redcord::ServerScripts
+  extend T::Sig
+
+  sig do
+    params(
+      key: T.any(String, Symbol),
+      args: T::Hash[T.untyped, T.untyped],
+    ).returns(Integer)
+  end
+  def create_hash_returning_id(key, args)
+    evalsha(
+      T.must(redcord_server_script_shas[:create_hash_returning_id]),
+      keys: [key],
+      argv: args.to_a.flatten,
+    ).to_i
+  end
+
+  sig do
+    params(
+      model: String,
+      id: Integer,
+      args: T::Hash[T.untyped, T.untyped],
+    ).void
+  end
+  def update_hash(model, id, args)
+    evalsha(
+      T.must(redcord_server_script_shas[:update_hash]),
+      keys: [model, id],
+      argv: args.to_a.flatten,
+    )
+  end
+
+  sig do
+    params(
+      model: String,
+      id: Integer
+    ).returns(Integer)
+  end
+  def delete_hash(model, id)
+    evalsha(
+      T.must(redcord_server_script_shas[:delete_hash]),
+      keys: [model, id]
+    )
+  end
+
+  sig do
+    params(
+      model: String,
+      query_conditions: T::Hash[T.untyped, T.untyped],
+      select_attrs: T::Set[Symbol]
+    ).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
+  end
+  def find_by_attr(model, query_conditions, select_attrs=Set.new)
+    res = evalsha(
+      T.must(redcord_server_script_shas[:find_by_attr]),
+      keys: [model] + query_conditions.to_a.flatten,
+      argv: select_attrs.to_a.flatten
+    )
+    # The Lua script will return this as a flattened array.
+    # Convert the result into a hash of {id -> model hash}
+    res_hash = res.each_slice(2)
+    res_hash.map { |key, val| [key.to_i, val.each_slice(2).to_h] }.to_h
+  end
+
+  sig do
+    params(
+      model: String,
+      query_conditions: T::Hash[T.untyped, T.untyped]
+    ).returns(Integer)
+  end
+  def find_by_attr_count(model, query_conditions)
+    evalsha(
+      T.must(redcord_server_script_shas[:find_by_attr_count]),
+      keys: [model] + query_conditions.to_a.flatten,
+    )
+  end
+end

data/lib/redcord/server_scripts/create_hash_returning_id.erb.lua

@@ -0,0 +1,68 @@
+--[[
+EVALSHA SHA1(__FILE__) [field value ...]
+> Time complexity: O(N) where N is the number of fields being set.
+
+Create a hash with the specified fields to their respective values stored at
+key when key does not exist.
+
+# Return value
+The id of the created hash as a string.
+--]]
+
+-- The arguments can be accessed by Lua using the KEYS global variable in the
+-- form of a one-based array (so KEYS[1], KEYS[2], ...).
+-- All the additional arguments should not represent key names and can be
+-- accessed by Lua using the ARGV global variable, very similarly to what
+-- happens with keys (so ARGV[1], ARGV[2], ...).
+
+--   KEYS[1] = Model.name
+--   ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
+<%= include_lua 'shared/lua_helper_methods' %>
+<%= include_lua 'shared/index_helper_methods' %>
+
+-- Validate input to script before making Redis db calls
+if #KEYS ~= 1 then
+  error('Expected keys to be of size 1')
+end
+if #ARGV % 2 ~= 0 then
+  error('Expected an even number of arguments')
+end
+
+local model = KEYS[1]
+
+-- Call the Redis command: INCR "#{Model.name}:id_seq". If "#{Model.name}:id_seq" does
+-- not exist, the command returns 0. It errors if the id_seq overflows a 64 bit
+-- signed integer.
+redis.call('incr', model .. ':id_seq')
+
+-- The Lua version used by Redis does not support 64 bit integers:
+--   https://github.com/antirez/redis/issues/5261
+-- We ignore the integer response from INCR and use the string response from
+-- the GET/MGET command.
+local id, ttl = unpack(redis.call('mget', model .. ':id_seq', model .. ':ttl'))
+local key = model .. ':id:' .. id
+
+-- Forward the script arguments to the Redis command HSET.
+-- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
+redis.call('hset', key, unpack(ARGV))
+
+-- Set TTL on key
+if ttl and ttl ~= '-1' then
+  redis.call('expire', key, ttl)
+end
+
+-- Add id value for any index and range index attributes
+local attrs_hash = to_hash(ARGV)
+local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
+if #index_attr_keys > 0 then
+  for _, attr_key in ipairs(index_attr_keys) do
+    add_id_to_index_attr(model, attr_key, attrs_hash[attr_key], id)
+  end
+end
+local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
+if #range_index_attr_keys > 0 then
+  for _, attr_key in ipairs(range_index_attr_keys) do
+    add_id_to_range_index_attr(model, attr_key, attrs_hash[attr_key], id)
+  end
+end
+return id

data/lib/redcord/server_scripts/delete_hash.erb.lua

@@ -0,0 +1,48 @@
+--[[
+EVALSHA SHA1(__FILE__) model id
+> Time complexity: O(1)
+
+Delete a hash at "#model:id:#id", and the corresponding id from the indexed 
+attribute id sets.
+
+# Return value
+The number of keys deleted from Redis
+--]]
+
+-- The arguments can be accessed by Lua using the KEYS global variable in the
+-- form of a one-based array (so KEYS[1], KEYS[2], ...).
+--
+--   KEYS[1] = Model.name
+--   KEYS[2] = id
+<%= include_lua 'shared/index_helper_methods' %>
+
+-- Validate input to script before making Redis db calls
+if #KEYS ~= 2 then
+  error('Expected keys of be of size 2')
+end
+
+local model = KEYS[1]
+local id = KEYS[2]
+
+-- key = "#{model}:id:{id}"
+local key = model .. ':id:' .. id
+
+-- Clean up id sets for both index and range index attributes
+local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
+if #index_attr_keys > 0 then
+-- Retrieve old index attr values so we can delete them in the attribute id sets
+  local attr_vals = redis.call('hmget', key, unpack(index_attr_keys))
+  for i=1, #index_attr_keys do
+    delete_id_from_index_attr(model, index_attr_keys[i], attr_vals[i], id)
+  end
+end
+local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
+if #range_index_attr_keys > 0 then
+  local attr_vals = redis.call('hmget', key, unpack(range_index_attr_keys))
+  for i=1, #range_index_attr_keys do
+    delete_id_from_range_index_attr(model, range_index_attr_keys[i], attr_vals[i], id)
+  end
+end
+
+-- delete the actual key
+return redis.call('del', key)

data/lib/redcord/server_scripts/find_by_attr.erb.lua

@@ -0,0 +1,67 @@
+--[[
+EVALSHA SHA1(__FILE__) model id
+> Time complexity: O(N) where N is the number of ids with these attributes
+
+Query for all model instances that have the given attribute values or value ranges.
+Return an error if an attribute is not an index.
+
+# Return value
+A hash of id:model of all the ids that match the query conditions given.
+--]]
+
+-- The arguments can be accessed by Lua using the KEYS global variable in the
+-- form of a one-based array (so KEYS[1], KEYS[2], ...).
+-- All the additional arguments should not represent key names and can be
+-- accessed by Lua using the ARGV global variable, very similarly to what
+-- happens with keys (so ARGV[1], ARGV[2], ...).
+--
+--   KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
+--   ARGV[1...N] = attr_key [attr_key ..]
+--
+--   For equality query conditions, key value pairs are expected to appear in
+--   the KEYS array as [attr_key, attr_val]
+--   For range query conditions, key value pairs are expected to appear in the
+--   KEYS array as [key min_val max_val]
+--
+--   The ARGV array is used to specify specific fields to select for each record. If
+--   the ARGV array is empty, then all fields will be retrieved.
+
+<%= include_lua 'shared/lua_helper_methods' %>
+<%= include_lua 'shared/query_helper_methods' %>
+
+if #KEYS < 3 then
+  error('Expected keys to be at least of size 3')
+end
+
+local model = KEYS[1]
+local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
+
+-- Get all ids which have the corresponding attribute values.
+local ids_set = nil
+-- For normal sets, Redis has SINTER built in to return the set intersection
+if #index_sets > 0 then
+   ids_set = to_set(redis.call('sinter', unpack(index_sets)))
+end
+-- For sorted sets, call helper function zinter_zrangebyscore, which calls
+-- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
+if #range_index_sets > 0 then
+  ids_set = intersect_range_index_sets(ids_set, range_index_sets)
+end
+
+-- Query for the hashes for all ids in the set intersection
+local res, stale_ids = unpack(batch_hget(model, ids_set, ARGV))
+
+-- Delete any stale ids which are no longer in redis from the id sets.
+-- This can happen if an entry was auto expired due to ttl, but not removed up yet
+-- from the id sets.
+if #stale_ids > 0 then
+  for _, key in ipairs(index_sets) do
+    redis.call('srem', key, unpack(stale_ids))
+  end
+  for _, key in ipairs(range_index_sets) do
+    local redis_key = key[1]
+    redis.call('zrem', redis_key, unpack(stale_ids))
+  end
+end
+
+return res

data/lib/redcord/server_scripts/find_by_attr_count.erb.lua

@@ -0,0 +1,52 @@
+--[[
+EVALSHA SHA1(__FILE__) model id
+> Time complexity: O(N) where N is the number of ids with these attributes
+
+Query for all model instances that have the given attribute values or value ranges.
+Return an error if an attribute is not an index.
+
+# Return value
+An integer number of records that match the query conditions given.
+--]]
+
+-- The arguments can be accessed by Lua using the KEYS global variable in the
+-- form of a one-based array (so KEYS[1], KEYS[2], ...).
+-- All the additional arguments should not represent key names and can be
+-- accessed by Lua using the ARGV global variable, very similarly to what
+-- happens with keys (so ARGV[1], ARGV[2], ...).
+--
+--   KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
+--
+--   For equality query conditions, key value pairs are expected to appear in
+--   the KEYS array as [attr_key, attr_val]
+--   For range query conditions, key value pairs are expected to appear in the
+--   KEYS array as [key min_val max_val]
+
+<%= include_lua 'shared/lua_helper_methods' %>
+<%= include_lua 'shared/query_helper_methods' %>
+
+if #KEYS < 3 then
+  error('Expected keys to be at least of size 3')
+end
+
+local model = KEYS[1]
+local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
+
+-- Get all ids which have the corresponding attribute values.
+local ids_set = nil
+-- For normal sets, Redis has SINTER built in to return the set intersection
+if #index_sets > 0 then
+   ids_set = to_set(redis.call('sinter', unpack(index_sets)))
+end
+-- For sorted sets, call helper function zinter_zrangebyscore, which calls
+-- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
+if #range_index_sets > 0 then
+  ids_set = intersect_range_index_sets(ids_set, range_index_sets)
+end
+
+-- Get the number of records which satisfy the query conditions.
+-- We do not delete stale ids as part of this function call because we do not have
+-- the list of ids which don't exist. The Redis command EXISTS key [key ...] is an O(1)
+-- operation that only returns the count of ids that exist. Getting the list of ids that
+-- don't exist would be an O(N) operation.
+return batch_exists(model, ids_set)