redcord 0.0.3 → 0.1.3

data/lib/redcord/serializer.rb

@@ -8,6 +8,8 @@ module Redcord
   # Raised by Model.where
   class AttributeNotIndexed < StandardError; end
   class WrongAttributeType < TypeError; end
+  class CustomIndexInvalidQuery < StandardError; end
+  class CustomIndexInvalidDesign < StandardError; end
 end
 
 # This module defines various helper methods on Redcord for serialization
@@ -31,50 +33,36 @@ module Redcord::Serializer
     sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
     def encode_attr_value(attribute, val)
       if !val.blank? && TIME_TYPES.include?(props[attribute][:type])
-        val = val.to_f
+        time_in_nano_sec = val.to_i * 1_000_000_000
+        time_in_nano_sec >= 0 ? time_in_nano_sec + val.nsec : time_in_nano_sec - val.nsec
+      elsif val.is_a?(Float)
+        # Encode as round-trippable float64
+        '%1.16e' % [val]
+      else
+        val
       end
-
-      val
     end
 
     sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
     def decode_attr_value(attribute, val)
       if !val.blank? && TIME_TYPES.include?(props[attribute][:type])
-        val = Time.zone.at(val.to_f)
-      end
+        val = val.to_i
+        nsec = val >= 0 ? val % 1_000_000_000 : -val % 1_000_000_000
 
-      val
+        Time.zone.at(val / 1_000_000_000).change(nsec: nsec)
+      else
+        val
+      end
     end
 
     sig { params(attr_key: Symbol, attr_val: T.untyped).returns(T.untyped)}
-    def validate_and_encode_query(attr_key, attr_val)
-      # Validate that attributes queried for are index attributes
-      if !class_variable_get(:@@index_attributes).include?(attr_key) &&
-         !class_variable_get(:@@range_index_attributes).include?(attr_key)
-        raise(
-          Redcord::AttributeNotIndexed,
-          "#{attr_key} is not an indexed attribute.",
-        )
-      end
-
-      # Validate attribute types for normal index attributes
+    def validate_types_and_encode_query(attr_key, attr_val)
+      # Validate attribute types for index attributes
       attr_type = get_attr_type(attr_key)
-      if class_variable_get(:@@index_attributes).include?(attr_key)
+      if class_variable_get(:@@index_attributes).include?(attr_key) || attr_key == shard_by_attribute
         validate_attr_type(attr_val, attr_type)
       else
-        # Validate attribute types for range index attributes
-        if attr_val.is_a?(Redcord::RangeInterval)
-          validate_attr_type(
-            attr_val.min,
-            T.cast(T.nilable(attr_type), T::Types::Base),
-          )
-          validate_attr_type(
-            attr_val.max,
-            T.cast(T.nilable(attr_type), T::Types::Base),
-          )
-        else
-          validate_attr_type(attr_val, attr_type)
-        end
+        validate_range_attr_types(attr_val, attr_type)
 
         # Range index attributes need to be further encoded into a format
         # understood by the Lua script.
@@ -82,10 +70,73 @@ module Redcord::Serializer
           attr_val = encode_range_index_attr_val(attr_key, attr_val)
         end
       end
-
       attr_val
     end
 
+    # Validate that attributes queried for are index attributes
+    # For custom index: validate that attributes are present in specified index
+    sig { params(attr_keys: T::Array[Symbol], custom_index_name: T.nilable(Symbol)).void}
+    def validate_index_attributes(attr_keys, custom_index_name: nil)
+      custom_index_attributes = class_variable_get(:@@custom_index_attributes)[custom_index_name]
+      attr_keys.each do |attr_key|
+        next if attr_key == shard_by_attribute
+
+        if !custom_index_attributes.empty?
+          if !custom_index_attributes.include?(attr_key)
+            raise(
+              Redcord::AttributeNotIndexed,
+              "#{attr_key} is not a part of #{custom_index_name} index.",
+            )
+          end
+        else
+          if !class_variable_get(:@@index_attributes).include?(attr_key) &&
+            !class_variable_get(:@@range_index_attributes).include?(attr_key)
+            raise(
+              Redcord::AttributeNotIndexed,
+              "#{attr_key} is not an indexed attribute.",
+            )
+          end
+        end
+      end
+    end
+
+    # Validate exclusive ranges not used; Change all query conditions to range form;
+    # The position of the attribute and type of query is validated on Lua side
+    sig { params(query_conditions: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped])}
+    def validate_and_adjust_custom_index_query_conditions(query_conditions)
+      adjusted_query_conditions = query_conditions.clone
+      query_conditions.each do |attr_key, condition|
+        if !condition.is_a?(Array)
+          adjusted_query_conditions[attr_key] = [condition, condition]
+        elsif condition[0].to_s[0] == '(' or condition[1].to_s[0] == '('
+          raise(Redcord::CustomIndexInvalidQuery, "Custom index doesn't support exclusive ranges")
+        end
+      end
+      adjusted_query_conditions
+    end
+
+    sig {
+      params(
+        attr_val: T.untyped,
+        attr_type: T.any(Class, T::Types::Base),
+      ).void
+    }
+    def validate_range_attr_types(attr_val, attr_type)
+      # Validate attribute types for range index attributes
+      if attr_val.is_a?(Redcord::RangeInterval)
+        validate_attr_type(
+          attr_val.min,
+          T.cast(T.nilable(attr_type), T::Types::Base),
+        )
+        validate_attr_type(
+          attr_val.max,
+          T.cast(T.nilable(attr_type), T::Types::Base),
+        )
+      else
+        validate_attr_type(attr_val, attr_type)
+      end
+    end
+
     sig {
       params(
         attr_val: T.untyped,
@@ -137,7 +188,7 @@ module Redcord::Serializer
     sig {
       params(
         redis_hash: T::Hash[T.untyped, T.untyped],
-        id: Integer,
+        id: String,
       ).returns(T.untyped)
     }
     def coerce_and_set_id(redis_hash, id)

data/lib/redcord/server_scripts/create_hash.erb.lua

@@ -0,0 +1,81 @@
+--[[
+EVALSHA SHA1(__FILE__) [field value ...]
+> Time complexity: O(N) where N is the number of fields being set.
+
+Create a hash with the specified fields to their respective values stored at
+key when key does not exist.
+
+# Return value
+The id of the created hash as a string.
+--]]
+
+-- The arguments can be accessed by Lua using the KEYS global variable in the
+-- form of a one-based array (so KEYS[1], KEYS[2], ...).
+-- All the additional arguments should not represent key names and can be
+-- accessed by Lua using the ARGV global variable, very similarly to what
+-- happens with keys (so ARGV[1], ARGV[2], ...).
+
+--   KEYS = id hash_tag
+--   ARGV = Model.name ttl index_attr_size range_index_attr_size custom_index_attrs_flat_size [index_attr_key ...] [range_index_attr_key ...]
+--          [custom_index_name attrs_size [custom_index_attr_key ...] ...] attr_key attr_val [attr_key attr_val ..]
+<%= include_lua 'shared/lua_helper_methods' %>
+<%= include_lua 'shared/index_helper_methods' %>
+
+-- Validate input to script before making Redis db calls
+if #KEYS ~= 2 then
+  error('Expected keys to be of size 2')
+end
+
+local id, hash_tag = unpack(KEYS)
+local model, ttl = unpack(ARGV)
+local key = model .. ':id:' .. id
+
+local index_attr_pos = 6
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+-- Starting position of the attr_key-attr_val pairs
+local attr_pos = custom_attr_pos + ARGV[5]
+
+
+if redis.call('exists', key) ~= 0 then
+  error(key .. ' already exists')
+end
+
+-- Forward the script arguments to the Redis command HSET.
+-- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
+redis.call('hset', key, unpack(ARGV, attr_pos))
+
+-- Set TTL on key
+if ttl and ttl ~= '-1' then
+  redis.call('expire', key, ttl)
+end
+
+-- Add id value for any index and range index attributes
+local attrs_hash = to_hash(unpack(ARGV, attr_pos))
+local index_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
+if #index_attr_keys > 0 then
+  for _, attr_key in ipairs(index_attr_keys) do
+    add_id_to_index_attr(hash_tag, model, attr_key, attrs_hash[attr_key], id)
+  end
+end
+local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}
+if #range_index_attr_keys > 0 then
+  for _, attr_key in ipairs(range_index_attr_keys) do
+    add_id_to_range_index_attr(hash_tag, model, attr_key, attrs_hash[attr_key], id)
+  end
+end
+
+-- Add a record to every custom index
+local custom_index_attr_keys = {unpack(ARGV, custom_attr_pos, attr_pos - 1)}
+local i = 1
+while i < #custom_index_attr_keys do
+  local index_name, attrs_num = custom_index_attr_keys[i], custom_index_attr_keys[i+1]
+  local attr_values = {}
+  for j, attr_key in ipairs({unpack(custom_index_attr_keys, i + 2, i + attrs_num + 1)}) do
+    attr_values[j] = attrs_hash[attr_key]
+  end
+  add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
+  i = i + 2 + attrs_num
+end
+
+return nil

data/lib/redcord/server_scripts/delete_hash.erb.lua

@@ -12,8 +12,8 @@ The number of keys deleted from Redis
 -- The arguments can be accessed by Lua using the KEYS global variable in the
 -- form of a one-based array (so KEYS[1], KEYS[2], ...).
 --
---   KEYS[1] = Model.name
---   KEYS[2] = id
+--   KEYS = id, hash_tag
+--   ARGV = Model.name index_attr_size range_index_attr_size [index_attr_key ...] [range_index_attr_key ...] [custom_index_name ...]
 <%= include_lua 'shared/index_helper_methods' %>
 
 -- Validate input to script before making Redis db calls
@@ -21,28 +21,37 @@ if #KEYS ~= 2 then
   error('Expected keys of be of size 2')
 end
 
-local model = KEYS[1]
-local id = KEYS[2]
+local model = ARGV[1]
+local id, hash_tag = unpack(KEYS)
 
 -- key = "#{model}:id:{id}"
 local key = model .. ':id:' .. id
 
+local index_attr_pos = 4
+local range_attr_pos = index_attr_pos + ARGV[2]
+local custom_index_pos = range_attr_pos + ARGV[3]
+
 -- Clean up id sets for both index and range index attributes
-local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
+local index_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
 if #index_attr_keys > 0 then
 -- Retrieve old index attr values so we can delete them in the attribute id sets
   local attr_vals = redis.call('hmget', key, unpack(index_attr_keys))
   for i=1, #index_attr_keys do
-    delete_id_from_index_attr(model, index_attr_keys[i], attr_vals[i], id)
+    delete_id_from_index_attr(hash_tag, model, index_attr_keys[i], attr_vals[i], id)
   end
 end
-local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
+local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_index_pos - 1)}
 if #range_index_attr_keys > 0 then
   local attr_vals = redis.call('hmget', key, unpack(range_index_attr_keys))
   for i=1, #range_index_attr_keys do
-    delete_id_from_range_index_attr(model, range_index_attr_keys[i], attr_vals[i], id)
+    delete_id_from_range_index_attr(hash_tag, model, range_index_attr_keys[i], attr_vals[i], id)
   end
 end
+-- Delete record from custom indexes
+local custom_index_names = {unpack(ARGV, custom_index_pos)}
+for _, index_name in ipairs(custom_index_names) do
+  delete_record_from_custom_index(hash_tag, model, index_name, id)
+end
 
 -- delete the actual key
 return redis.call('del', key)

data/lib/redcord/server_scripts/find_by_attr.erb.lua

@@ -15,9 +15,11 @@ A hash of id:model of all the ids that match the query conditions given.
 -- accessed by Lua using the ARGV global variable, very similarly to what
 -- happens with keys (so ARGV[1], ARGV[2], ...).
 --
---   KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
---   ARGV[1...N] = attr_key [attr_key ..]
---
+--   KEYS[1] = hash_tag
+--   ARGV = Model.name custom_index_name num_index_attr num_range_index_attr num_custom_index_attr num_query_conditions 
+--          [index_attrs ...] [range_index_attrs ...] [custom_index_attrs ...] [query_conidtions ...] [attr_selections ...]
+--          [query_conidtions ...]: [attr_key1 attr_val1 attr_key2 attr_val2 ...]
+--          [attr_selections ...]: [attr_key1 attr_key2 ...]
 --   For equality query conditions, key value pairs are expected to appear in
 --   the KEYS array as [attr_key, attr_val]
 --   For range query conditions, key value pairs are expected to appear in the
@@ -29,27 +31,60 @@ A hash of id:model of all the ids that match the query conditions given.
 <%= include_lua 'shared/lua_helper_methods' %>
 <%= include_lua 'shared/query_helper_methods' %>
 
-if #KEYS < 3 then
-  error('Expected keys to be at least of size 3')
+if #KEYS ~=1 then
+  error('Expected keys to be of size 1')
 end
 
-local model = KEYS[1]
-local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
+local model = ARGV[1]
+
+local index_name = ARGV[2]
+local index_attr_pos = 7
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+local query_cond_pos = custom_attr_pos + ARGV[5]
+local attr_selection_pos = query_cond_pos + ARGV[6]
 
 -- Get all ids which have the corresponding attribute values.
 local ids_set = nil
--- For normal sets, Redis has SINTER built in to return the set intersection
-if #index_sets > 0 then
-   ids_set = to_set(redis.call('sinter', unpack(index_sets)))
-end
--- For sorted sets, call helper function zinter_zrangebyscore, which calls
--- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
-if #range_index_sets > 0 then
-  ids_set = intersect_range_index_sets(ids_set, range_index_sets)
+local index_sets, range_index_sets = {}, {}
+
+-- If custom index name is empty -> use single-attribute indices
+if index_name == '' then
+  index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(
+    KEYS[1],
+    model,
+    to_set({unpack(ARGV, index_attr_pos, range_attr_pos - 1)}),
+    to_set({unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}),
+    unpack(ARGV, query_cond_pos, attr_selection_pos - 1)
+  ))
+
+  -- For normal sets, Redis has SINTER built in to return the set intersection
+  if #index_sets > 0 then
+    ids_set = to_set(redis.call('sinter', unpack(index_sets)))
+  end
+  -- For sorted sets, call helper function zinter_zrangebyscore, which calls
+  -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
+  if #range_index_sets > 0 then
+    ids_set = intersect_range_index_sets(ids_set, range_index_sets)
+  end
+else
+  local custom_index_attrs = {unpack(ARGV, custom_attr_pos, query_cond_pos - 1)}
+  local custom_index_query = validate_and_parse_query_conditions_custom(
+    KEYS[1],
+    model,
+    index_name,
+    custom_index_attrs,
+    {unpack(ARGV, query_cond_pos, attr_selection_pos - 1)}
+  )
+  if #custom_index_query > 0 then
+    ids_set = get_id_set_from_custom_index(ids_set, custom_index_query)
+  else
+    ids_set = {}
+  end
 end
 
 -- Query for the hashes for all ids in the set intersection
-local res, stale_ids = unpack(batch_hget(model, ids_set, ARGV))
+local res, stale_ids = unpack(batch_hget(model, ids_set, unpack(ARGV, attr_selection_pos)))
 
 -- Delete any stale ids which are no longer in redis from the id sets.
 -- This can happen if an entry was auto expired due to ttl, but not removed up yet

data/lib/redcord/server_scripts/find_by_attr_count.erb.lua

@@ -15,7 +15,9 @@ An integer number of records that match the query conditions given.
 -- accessed by Lua using the ARGV global variable, very similarly to what
 -- happens with keys (so ARGV[1], ARGV[2], ...).
 --
---   KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
+--   KEYS[1] = hash_tag
+--   ARGV = Model.name custom_index_name num_index_attr num_range_index_attr num_custom_index_attr
+--          [index_attrs ...] [range_index_attrs ...] [custom_index_attrs ...] [query_conidtions ...]
 --
 --   For equality query conditions, key value pairs are expected to appear in
 --   the KEYS array as [attr_key, attr_val]
@@ -25,25 +27,54 @@ An integer number of records that match the query conditions given.
 <%= include_lua 'shared/lua_helper_methods' %>
 <%= include_lua 'shared/query_helper_methods' %>
 
-if #KEYS < 3 then
-  error('Expected keys to be at least of size 3')
+if #KEYS ~=1 then
+  error('Expected keys to be of size 1')
 end
 
-local model = KEYS[1]
-local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
+local model = ARGV[1]
+
+local index_name = ARGV[2]
+local index_attr_pos = 6
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+local query_cond_pos = custom_attr_pos + ARGV[5]
 
 -- Get all ids which have the corresponding attribute values.
 local ids_set = nil
--- For normal sets, Redis has SINTER built in to return the set intersection
-if #index_sets > 0 then
-   ids_set = to_set(redis.call('sinter', unpack(index_sets)))
-end
--- For sorted sets, call helper function zinter_zrangebyscore, which calls
--- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
-if #range_index_sets > 0 then
-  ids_set = intersect_range_index_sets(ids_set, range_index_sets)
-end
 
+if index_name == '' then
+  local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(
+    KEYS[1],
+    model,
+    to_set({unpack(ARGV, index_attr_pos, range_attr_pos - 1)}),
+    to_set({unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}),
+    unpack(ARGV, query_cond_pos)
+  ))
+
+  -- For normal sets, Redis has SINTER built in to return the set intersection
+  if #index_sets > 0 then
+    ids_set = to_set(redis.call('sinter', unpack(index_sets)))
+  end
+  -- For sorted sets, call helper function zinter_zrangebyscore, which calls
+  -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
+  if #range_index_sets > 0 then
+    ids_set = intersect_range_index_sets(ids_set, range_index_sets)
+  end
+else
+  local custom_index_attrs = {unpack(ARGV, custom_attr_pos, query_cond_pos - 1)}
+  local custom_index_query = validate_and_parse_query_conditions_custom(
+    KEYS[1],
+    model,
+    index_name,
+    custom_index_attrs,
+    {unpack(ARGV, query_cond_pos)}
+  )
+  if #custom_index_query > 0 then
+    ids_set = get_id_set_from_custom_index(ids_set, custom_index_query)
+  else
+    ids_set = {}
+  end
+end
 -- Get the number of records which satisfy the query conditions.
 -- We do not delete stale ids as part of this function call because we do not have
 -- the list of ids which don't exist. The Redis command EXISTS key [key ...] is an O(1)