redcord 0.0.4 → 0.1.0

data/lib/redcord/redis_connection.rb

@@ -5,7 +5,7 @@
 require 'rails'
 
 require 'redcord/lua_script_reader'
-require 'redcord/prepared_redis'
+require 'redcord/redis'
 
 module Redcord::RedisConnection
   extend T::Sig
@@ -29,17 +29,17 @@ module Redcord::RedisConnection
       (env_config[name.underscore] || env_config['default']).symbolize_keys
     end
 
-    sig { returns(Redcord::PreparedRedis) }
+    sig { returns(Redcord::Redis) }
     def redis
       Redcord::RedisConnection.connections[name.underscore] ||= prepare_redis!
     end
 
-    sig { returns(Redcord::PreparedRedis) }
+    sig { returns(Redcord::Redis) }
     def establish_connection
       Redcord::RedisConnection.connections[name.underscore] = prepare_redis!
     end
 
-    sig { params(redis: Redis).returns(Redcord::PreparedRedis) }
+    sig { params(redis: Redis).returns(Redcord::Redis) }
     def redis=(redis)
       Redcord::RedisConnection.connections[name.underscore] =
         prepare_redis!(redis)
@@ -50,11 +50,11 @@ module Redcord::RedisConnection
     # definitions in each Redis query.
     #
     # TODO: Replace this with Redcord migrations
-    sig { params(client: T.nilable(Redis)).returns(Redcord::PreparedRedis) }
+    sig { params(client: T.nilable(Redis)).returns(Redcord::Redis) }
     def prepare_redis!(client = nil)
-      return client if client.is_a?(Redcord::PreparedRedis)
+      return client if client.is_a?(Redcord::Redis)
 
-      client = Redcord::PreparedRedis.new(
+      client = Redcord::Redis.new(
         **(
           if client.nil?
             connection_config
@@ -65,12 +65,7 @@ module Redcord::RedisConnection
         logger: Redcord::Logger.proxy,
       )
 
-      client.pipelined do
-        Redcord::RedisConnection.procs_to_prepare.each do |proc_to_prepare|
-          proc_to_prepare.call(client)
-        end
-      end
-
+      client.ping
       client
     end
   end
@@ -78,7 +73,7 @@ module Redcord::RedisConnection
   module InstanceMethods
     extend T::Sig
 
-    sig { returns(Redcord::PreparedRedis) }
+    sig { returns(Redcord::Redis) }
     def redis
       self.class.redis
     end

data/lib/redcord/relation.rb

@@ -5,42 +5,62 @@
 require 'active_support/core_ext/array'
 require 'active_support/core_ext/module'
 
+module Redcord
+  class InvalidQuery < StandardError; end
+end
+
 class Redcord::Relation
   extend T::Sig
 
   sig { returns(T.class_of(Redcord::Base)) }
   attr_reader :model
 
-  sig { returns(T::Hash[Symbol, T.untyped]) }
-  attr_reader :query_conditions
-
   sig { returns(T::Set[Symbol]) }
   attr_reader :select_attrs
 
+  sig { returns(T.nilable(Symbol)) }
+  attr_reader :custom_index_name
+
+  sig { returns(T::Hash[Symbol, T.untyped]) }
+  attr_reader :regular_index_query_conditions
+
+  sig { returns(T::Hash[Symbol, T.untyped]) }
+  attr_reader :custom_index_query_conditions
+
   sig do
     params(
       model: T.class_of(Redcord::Base),
-      query_conditions: T::Hash[Symbol, T.untyped],
+      regular_index_query_conditions: T::Hash[Symbol, T.untyped],
+      custom_index_query_conditions: T::Hash[Symbol, T.untyped],
       select_attrs: T::Set[Symbol],
+      custom_index_name: T.nilable(Symbol)
     ).void
   end
   def initialize(
     model,
-    query_conditions = {},
-    select_attrs = Set.new
+    regular_index_query_conditions = {},
+    custom_index_query_conditions = {},
+    select_attrs = Set.new,
+    custom_index_name: nil
   )
     @model = model
-    @query_conditions = query_conditions
+    @regular_index_query_conditions = regular_index_query_conditions
+    @custom_index_query_conditions = custom_index_query_conditions
     @select_attrs = select_attrs
+    @custom_index_name = custom_index_name
   end
 
   sig { params(args: T::Hash[Symbol, T.untyped]).returns(Redcord::Relation) }
   def where(args)
     encoded_args = args.map do |attr_key, attr_val|
-      encoded_val = model.validate_and_encode_query(attr_key, attr_val)
+      encoded_val = model.validate_types_and_encode_query(attr_key, attr_val)
       [attr_key, encoded_val]
     end
-    query_conditions.merge!(encoded_args.to_h)
+
+    regular_index_query_conditions.merge!(encoded_args.to_h)
+    if custom_index_name
+      with_index(custom_index_name)
+    end
     self
   end
 
@@ -68,7 +88,24 @@ class Redcord::Relation
 
   sig { returns(Integer) }
   def count
-    redis.find_by_attr_count(model.model_key, query_conditions)
+    model.validate_index_attributes(query_conditions.keys, custom_index_name: custom_index_name)
+    redis.find_by_attr_count(
+      model.model_key,
+      extract_query_conditions!,
+      index_attrs: model._script_arg_index_attrs,
+      range_index_attrs: model._script_arg_range_index_attrs,
+      custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
+      hash_tag: extract_hash_tag!,
+      custom_index_name: custom_index_name
+    )
+  end
+
+  sig { params(index_name: T.nilable(Symbol)).returns(Redcord::Relation) }
+  def with_index(index_name)
+    @custom_index_name = index_name
+    adjusted_query_conditions = model.validate_and_adjust_custom_index_query_conditions(regular_index_query_conditions)
+    custom_index_query_conditions.merge!(adjusted_query_conditions)
+    self
   end
 
   delegate(
@@ -137,17 +174,52 @@ class Redcord::Relation
 
   private
 
+  sig { returns(T.nilable(String)) }
+  def extract_hash_tag!
+    attr = model.shard_by_attribute
+    return nil if attr.nil?
+
+    if !query_conditions.keys.include?(attr)
+      raise(
+        Redcord::InvalidQuery,
+        "Queries must contain attribute '#{attr}' since model #{model.name} is sharded by this attribute"
+      )
+    end
+
+    # Query conditions on custom index are always in form of range, even when query is by value condition is [value_x, value_x]
+    # When in fact query is by value, range is trasformed to a single value to pass the validation.
+    condition = query_conditions[attr]
+    if custom_index_name and condition.first == condition.last
+      condition = condition.first
+    end
+    case condition
+    when Integer, String
+      "{#{condition}}"
+    else
+      raise(
+        Redcord::InvalidQuery,
+        "Does not support query condition #{condition} on a Redis Cluster",
+      )
+    end
+  end
+
   sig { returns(T::Array[T.untyped]) }
   def execute_query
     Redcord::Base.trace(
      'redcord_relation_execute_query',
      model_name: model.name,
     ) do
+      model.validate_index_attributes(query_conditions.keys, custom_index_name: custom_index_name)
       if !select_attrs.empty?
         res_hash = redis.find_by_attr(
           model.model_key,
-          query_conditions,
-          select_attrs,
+          extract_query_conditions!,
+          select_attrs: select_attrs,
+          index_attrs: model._script_arg_index_attrs,
+          range_index_attrs: model._script_arg_range_index_attrs,
+          custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
+          hash_tag: extract_hash_tag!,
+          custom_index_name: custom_index_name
         )
 
         res_hash.map do |id, args|
@@ -158,7 +230,12 @@ class Redcord::Relation
       else
         res_hash = redis.find_by_attr(
           model.model_key,
-          query_conditions,
+          extract_query_conditions!,
+          index_attrs: model._script_arg_index_attrs,
+          range_index_attrs: model._script_arg_range_index_attrs,
+          custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
+          hash_tag: extract_hash_tag!,
+          custom_index_name: custom_index_name
         )
 
         res_hash.map { |id, args| model.coerce_and_set_id(args, id) }
@@ -166,8 +243,24 @@ class Redcord::Relation
     end
   end
 
-  sig { returns(Redcord::PreparedRedis) }
+  sig { returns(Redcord::Redis) }
   def redis
     model.redis
   end
+
+  sig { returns(T::Hash[Symbol, T.untyped]) }
+  def query_conditions
+    custom_index_name ? custom_index_query_conditions : regular_index_query_conditions
+  end
+
+  sig { returns(T::Hash[Symbol, T.untyped]) }
+  def extract_query_conditions!
+    attr = model.shard_by_attribute
+    return query_conditions if attr.nil?
+
+    cond = query_conditions.reject { |key| key == attr }
+    raise Redcord::InvalidQuery, "Cannot query only by shard_by_attribute: #{attr}" if cond.empty?
+
+    cond
+  end
 end

data/lib/redcord/serializer.rb

@@ -8,6 +8,8 @@ module Redcord
   # Raised by Model.where
   class AttributeNotIndexed < StandardError; end
   class WrongAttributeType < TypeError; end
+  class CustomIndexInvalidQuery < StandardError; end
+  class CustomIndexInvalidDesign < StandardError; end
 end
 
 # This module defines various helper methods on Redcord for serialization
@@ -31,50 +33,36 @@ module Redcord::Serializer
     sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
     def encode_attr_value(attribute, val)
       if !val.blank? && TIME_TYPES.include?(props[attribute][:type])
-        val = val.to_f
+        time_in_nano_sec = val.to_i * 1_000_000_000
+        time_in_nano_sec >= 0 ? time_in_nano_sec + val.nsec : time_in_nano_sec - val.nsec
+      elsif val.is_a?(Float)
+        # Encode as round-trippable float64
+        '%1.16e' % [val]
+      else
+        val
       end
-
-      val
     end
 
     sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
     def decode_attr_value(attribute, val)
       if !val.blank? && TIME_TYPES.include?(props[attribute][:type])
-        val = Time.zone.at(val.to_f)
-      end
+        val = val.to_i
+        nsec = val >= 0 ? val % 1_000_000_000 : -val % 1_000_000_000
 
-      val
+        Time.zone.at(val / 1_000_000_000).change(nsec: nsec)
+      else
+        val
+      end
     end
 
     sig { params(attr_key: Symbol, attr_val: T.untyped).returns(T.untyped)}
-    def validate_and_encode_query(attr_key, attr_val)
-      # Validate that attributes queried for are index attributes
-      if !class_variable_get(:@@index_attributes).include?(attr_key) &&
-         !class_variable_get(:@@range_index_attributes).include?(attr_key)
-        raise(
-          Redcord::AttributeNotIndexed,
-          "#{attr_key} is not an indexed attribute.",
-        )
-      end
-
-      # Validate attribute types for normal index attributes
+    def validate_types_and_encode_query(attr_key, attr_val)
+      # Validate attribute types for index attributes
       attr_type = get_attr_type(attr_key)
-      if class_variable_get(:@@index_attributes).include?(attr_key)
+      if class_variable_get(:@@index_attributes).include?(attr_key) || attr_key == shard_by_attribute
         validate_attr_type(attr_val, attr_type)
       else
-        # Validate attribute types for range index attributes
-        if attr_val.is_a?(Redcord::RangeInterval)
-          validate_attr_type(
-            attr_val.min,
-            T.cast(T.nilable(attr_type), T::Types::Base),
-          )
-          validate_attr_type(
-            attr_val.max,
-            T.cast(T.nilable(attr_type), T::Types::Base),
-          )
-        else
-          validate_attr_type(attr_val, attr_type)
-        end
+        validate_range_attr_types(attr_val, attr_type)
 
         # Range index attributes need to be further encoded into a format
         # understood by the Lua script.
@@ -82,10 +70,73 @@ module Redcord::Serializer
           attr_val = encode_range_index_attr_val(attr_key, attr_val)
         end
       end
-
       attr_val
     end
 
+    # Validate that attributes queried for are index attributes
+    # For custom index: validate that attributes are present in specified index
+    sig { params(attr_keys: T::Array[Symbol], custom_index_name: T.nilable(Symbol)).void}
+    def validate_index_attributes(attr_keys, custom_index_name: nil)
+      custom_index_attributes = class_variable_get(:@@custom_index_attributes)[custom_index_name]
+      attr_keys.each do |attr_key|
+        next if attr_key == shard_by_attribute
+
+        if !custom_index_attributes.empty?
+          if !custom_index_attributes.include?(attr_key)
+            raise(
+              Redcord::AttributeNotIndexed,
+              "#{attr_key} is not a part of #{custom_index_name} index.",
+            )
+          end
+        else
+          if !class_variable_get(:@@index_attributes).include?(attr_key) &&
+            !class_variable_get(:@@range_index_attributes).include?(attr_key)
+            raise(
+              Redcord::AttributeNotIndexed,
+              "#{attr_key} is not an indexed attribute.",
+            )
+          end
+        end
+      end
+    end
+
+    # Validate exclusive ranges not used; Change all query conditions to range form;
+    # The position of the attribute and type of query is validated on Lua side
+    sig { params(query_conditions: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped])}
+    def validate_and_adjust_custom_index_query_conditions(query_conditions)
+      adjusted_query_conditions = query_conditions.clone
+      query_conditions.each do |attr_key, condition|
+        if !condition.is_a?(Array)
+          adjusted_query_conditions[attr_key] = [condition, condition]
+        elsif condition[0].to_s[0] == '(' or condition[1].to_s[0] == '('
+          raise(Redcord::CustomIndexInvalidQuery, "Custom index doesn't support exclusive ranges")
+        end
+      end
+      adjusted_query_conditions
+    end
+
+    sig {
+      params(
+        attr_val: T.untyped,
+        attr_type: T.any(Class, T::Types::Base),
+      ).void
+    }
+    def validate_range_attr_types(attr_val, attr_type)
+      # Validate attribute types for range index attributes
+      if attr_val.is_a?(Redcord::RangeInterval)
+        validate_attr_type(
+          attr_val.min,
+          T.cast(T.nilable(attr_type), T::Types::Base),
+        )
+        validate_attr_type(
+          attr_val.max,
+          T.cast(T.nilable(attr_type), T::Types::Base),
+        )
+      else
+        validate_attr_type(attr_val, attr_type)
+      end
+    end
+
     sig {
       params(
         attr_val: T.untyped,
@@ -137,7 +188,7 @@ module Redcord::Serializer
     sig {
       params(
         redis_hash: T::Hash[T.untyped, T.untyped],
-        id: Integer,
+        id: String,
       ).returns(T.untyped)
     }
     def coerce_and_set_id(redis_hash, id)

data/lib/redcord/server_scripts/create_hash.erb.lua

@@ -0,0 +1,81 @@
+--[[
+EVALSHA SHA1(__FILE__) [field value ...]
+> Time complexity: O(N) where N is the number of fields being set.
+
+Create a hash with the specified fields to their respective values stored at
+key when key does not exist.
+
+# Return value
+The id of the created hash as a string.
+--]]
+
+-- The arguments can be accessed by Lua using the KEYS global variable in the
+-- form of a one-based array (so KEYS[1], KEYS[2], ...).
+-- All the additional arguments should not represent key names and can be
+-- accessed by Lua using the ARGV global variable, very similarly to what
+-- happens with keys (so ARGV[1], ARGV[2], ...).
+
+--   KEYS = id hash_tag
+--   ARGV = Model.name ttl index_attr_size range_index_attr_size custom_index_attrs_flat_size [index_attr_key ...] [range_index_attr_key ...]
+--          [custom_index_name attrs_size [custom_index_attr_key ...] ...] attr_key attr_val [attr_key attr_val ..]
+<%= include_lua 'shared/lua_helper_methods' %>
+<%= include_lua 'shared/index_helper_methods' %>
+
+-- Validate input to script before making Redis db calls
+if #KEYS ~= 2 then
+  error('Expected keys to be of size 2')
+end
+
+local id, hash_tag = unpack(KEYS)
+local model, ttl = unpack(ARGV)
+local key = model .. ':id:' .. id
+
+local index_attr_pos = 6
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+-- Starting position of the attr_key-attr_val pairs
+local attr_pos = custom_attr_pos + ARGV[5]
+
+
+if redis.call('exists', key) ~= 0 then
+  error(key .. ' already exists')
+end
+
+-- Forward the script arguments to the Redis command HSET.
+-- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
+redis.call('hset', key, unpack(ARGV, attr_pos))
+
+-- Set TTL on key
+if ttl and ttl ~= '-1' then
+  redis.call('expire', key, ttl)
+end
+
+-- Add id value for any index and range index attributes
+local attrs_hash = to_hash(unpack(ARGV, attr_pos))
+local index_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
+if #index_attr_keys > 0 then
+  for _, attr_key in ipairs(index_attr_keys) do
+    add_id_to_index_attr(hash_tag, model, attr_key, attrs_hash[attr_key], id)
+  end
+end
+local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}
+if #range_index_attr_keys > 0 then
+  for _, attr_key in ipairs(range_index_attr_keys) do
+    add_id_to_range_index_attr(hash_tag, model, attr_key, attrs_hash[attr_key], id)
+  end
+end
+
+-- Add a record to every custom index
+local custom_index_attr_keys = {unpack(ARGV, custom_attr_pos, attr_pos - 1)}
+local i = 1
+while i < #custom_index_attr_keys do
+  local index_name, attrs_num = custom_index_attr_keys[i], custom_index_attr_keys[i+1]
+  local attr_values = {}
+  for j, attr_key in ipairs({unpack(custom_index_attr_keys, i + 2, i + attrs_num + 1)}) do
+    attr_values[j] = attrs_hash[attr_key]
+  end
+  add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
+  i = i + 2 + attrs_num
+end
+
+return nil